(CAT-2193): Fixed kubernetes environment setup for Debian. This allow…

…s spec tests to run successfully for debian platform

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']}
+      matrix: {'platform':['rhel-8', 'debian-11'],'collection':['puppet7-nightly', 'puppet8-nightly']}
 
     steps:
     - name: Checkout Source

.github/workflows/nightly.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']}
+      matrix: {'platform':['rhel-8', 'debian-11'],'collection':['puppet7-nightly', 'puppet8-nightly']}
 
     steps:
     - name: Checkout Source

manifests/repos.pp

@@ -64,12 +64,13 @@
       'Debian': {
         $codename = fact('os.distro.codename')
         apt::source { 'kubernetes':
-          location => pick($kubernetes_apt_location, 'https://apt.kubernetes.io'),
-          repos    => pick($kubernetes_apt_repos, 'main'),
-          release  => pick($kubernetes_apt_release, 'kubernetes-xenial'),
+          location => pick($kubernetes_apt_location, 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/'),
+          repos    => pick($kubernetes_apt_repos, ' '),
+          release  => pick($kubernetes_apt_release, ' /'),
+          comment  => 'Kubernetes',
           key      => {
-            'id'     => pick($kubernetes_key_id, 'A362B822F6DEDC652817EA46B53DC80D13EDEF05'),
-            'source' => pick($kubernetes_key_source, 'https://packages.cloud.google.com/apt/doc/apt-key.gpg'),
+            'name'   => 'kubernetes-apt-keyring.gpg',
+            'source' => pick($kubernetes_key_source, 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key'),
           },
         }
 
@@ -78,7 +79,7 @@
           apt::source { 'docker':
             location => pick($docker_apt_location, 'https://download.docker.com/linux/ubuntu/'),
             repos    => pick($docker_apt_repos, 'stable'),
-            release  => pick($docker_apt_release,$codename),
+            release  => pick($docker_apt_release, $codename),
             key      => {
               'id'     => pick($docker_key_id, '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'),
               'source' => pick($docker_key_source, 'https://download.docker.com/linux/ubuntu/gpg'),

spec/acceptance/kubernetes_spec.rb

@@ -30,12 +30,12 @@ class {'kubernetes':
             }
           }
           /^(Debian|Ubuntu)$/: {
-          class {'kubernetes':
-                  controller => true,
-                  schedule_on_controller => true,
-                  environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
-                  ignore_preflight_errors => ['NumCPU'],
-                }
+            class {'kubernetes':
+              controller => true,
+              schedule_on_controller => true,
+              environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
+              ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
+            }
           }
           default:  {
             class {'kubernetes': } # any other OS are not supported

spec/classes/repos_spec.rb

@@ -22,11 +22,11 @@
     let(:params) do
       {
         'container_runtime' => 'docker',
-        'kubernetes_apt_location' => 'http://apt.kubernetes.io',
-        'kubernetes_apt_release' => 'kubernetes-xenial',
-        'kubernetes_apt_repos' => 'main',
+        'kubernetes_apt_location' => 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/',
+        'kubernetes_apt_release' => ' /',
+        'kubernetes_apt_repos' => ' ',
         'kubernetes_key_id' => '54A647F9048D5688D7DA2ABE6A030B21BA07F4FB',
-        'kubernetes_key_source' => 'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
+        'kubernetes_key_source' => 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key',
         'kubernetes_yum_baseurl' => 'https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64',
         'kubernetes_yum_gpgkey' => 'https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg',
         'docker_apt_location' => 'https://download.docker.com/linux/ubuntu',
@@ -45,10 +45,10 @@
     it {
       expect(subject).to contain_apt__source('kubernetes').with(
         ensure: 'present',
-        location: 'http://apt.kubernetes.io',
-        repos: 'main',
-        release: 'kubernetes-xenial',
-        key: { 'id' => '54A647F9048D5688D7DA2ABE6A030B21BA07F4FB', 'source' => 'https://packages.cloud.google.com/apt/doc/apt-key.gpg' },
+        location: 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/',
+        repos: ' ',
+        release: ' /',
+        key: { 'name' => 'kubernetes-apt-keyring.gpg', 'source' => 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key' },
       )
     }
 
@@ -83,11 +83,11 @@
     let(:params) do
       {
         'container_runtime' => 'cri_containerd',
-        'kubernetes_apt_location' => 'http://apt.kubernetes.io',
-        'kubernetes_apt_release' => 'kubernetes-xenial',
-        'kubernetes_apt_repos' => 'main',
+        'kubernetes_apt_location' => 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/',
+        'kubernetes_apt_release' => ' /',
+        'kubernetes_apt_repos' => ' ',
         'kubernetes_key_id' => '54A647F9048D5688D7DA2ABE6A030B21BA07F4FB',
-        'kubernetes_key_source' => 'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
+        'kubernetes_key_source' => 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key',
         'kubernetes_yum_baseurl' => 'https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64',
         'kubernetes_yum_gpgkey' => 'https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg',
         'docker_apt_location' => 'https://download.docker.com/linux/ubuntu',
@@ -106,10 +106,10 @@
     it {
       expect(subject).to contain_apt__source('kubernetes').with(
         ensure: 'present',
-        location: 'http://apt.kubernetes.io',
-        repos: 'main',
-        release: 'kubernetes-xenial',
-        key: { 'id' => '54A647F9048D5688D7DA2ABE6A030B21BA07F4FB', 'source' => 'https://packages.cloud.google.com/apt/doc/apt-key.gpg' },
+        location: 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/',
+        repos: ' ',
+        release: ' /',
+        key: { 'name' => 'kubernetes-apt-keyring.gpg', 'source' => 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key' },
       )
     }
 

spec/spec_helper_acceptance_local.rb

@@ -40,12 +40,7 @@ def fetch_ip_hostname_by_role(role)
   platform = fetch_platform_by_node(ipaddr)
   ENV['TARGET_HOST'] = target_roles(role)[0][:name]
   hostname = run_shell('hostname').stdout.strip
-  os_family = run_shell("facter -y os.family | cut -d':' -f2 | tr -d ' '").stdout.strip
-  int_ipaddr = if os_family.casecmp('redhat').zero?
-                 run_shell("ip route get 8.8.8.8 | awk '{print $7; exit}'").stdout.strip
-               else
-                 run_shell("ip route get 8.8.8.8 | awk '{print $NF; exit}'").stdout.strip
-               end
+  int_ipaddr = run_shell("ip route get 8.8.8.8 | awk '{print $7; exit}'").stdout.strip
   [hostname, ipaddr, int_ipaddr]
 end
 
@@ -54,6 +49,10 @@ def change_target_host(role)
   ENV['TARGET_HOST'] = target_roles(role)[0][:name]
 end
 
+def os_family
+  run_shell("facter -y os.family | cut -d':' -f2 | tr -d ' '").stdout.strip
+end
+
 def reset_target_host
   ENV['TARGET_HOST'] = @orig_target_host
 end
@@ -70,37 +69,79 @@ def configure_puppet_server(controller, worker1, worker2)
   configure_puppet_agent('worker2')
   puppet_cert_sign
   # Create site.pp
-  site_pp = <<-EOS
-  node /#{controller[0]}/ {
-    class {'kubernetes':
-      kubernetes_version => '1.28.15',
-      kubernetes_package_version => '1.28.15',
-      controller_address => "#{controller[1]}:6443",
-      container_runtime => 'docker',
-      manage_docker => false,
-      controller => true,
-      schedule_on_controller => true,
-      environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
-      ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
-      cgroup_driver => 'systemd',
-      service_cidr => '10.138.0.0/12',
-    }
-  }
-  node /#{worker1}/ {
-    class {'kubernetes':
-      worker => true,
-      manage_docker => false,
-      cgroup_driver => 'systemd',
-    }
-  }
-  node /#{worker2}/  {
-    class {'kubernetes':
-      worker => true,
-      manage_docker => false,
-      cgroup_driver => 'systemd',
-    }
-  }
-  EOS
+
+  site_pp = if os_family.casecmp('redhat').zero?
+              <<-EOS
+                node /#{controller[0]}/ {
+                  class {'kubernetes':
+                    kubernetes_version => '1.28.15',
+                    kubernetes_package_version => '1.28.15',
+                    controller_address => "#{controller[1]}:6443",
+                    container_runtime => 'docker',
+                    manage_docker => false,
+                    controller => true,
+                    schedule_on_controller => true,
+                    environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
+                    ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
+                    cgroup_driver => 'systemd',
+                    service_cidr => '10.138.0.0/12',
+                  }
+                }
+
+                node /#{worker1}/ {
+                  class {'kubernetes':
+                    worker => true,
+                    manage_docker => false,
+                    cgroup_driver => 'systemd',
+                  }
+                }
+
+                node /#{worker2}/  {
+                  class {'kubernetes':
+                    worker => true,
+                    manage_docker => false,
+                    cgroup_driver => 'systemd',
+                  }
+                }
+              EOS
+            else
+              <<-EOS
+                node /#{controller[0]}/ {
+                  class {'kubernetes':
+                    kubernetes_version => '1.28.15',
+                    kubernetes_package_version => '1.28.15-1.1',
+                    controller_address => "#{controller[1]}:6443",
+                    container_runtime => 'cri_containerd',
+                    manage_docker => false,
+                    controller => true,
+                    schedule_on_controller => true,
+                    environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
+                    ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
+                    cgroup_driver => 'systemd',
+                    service_cidr => '10.138.0.0/12',
+                  }
+                }
+
+                node /#{worker1}/ {
+                  class {'kubernetes':
+                    kubernetes_package_version => '1.28.15-1.1',
+                    worker => true,
+                    manage_docker => false,
+                    cgroup_driver => 'systemd',
+                  }
+                }
+
+                node /#{worker2}/  {
+                  class {'kubernetes':
+                    kubernetes_package_version => '1.28.15-1.1',
+                    worker => true,
+                    manage_docker => false,
+                    cgroup_driver => 'systemd',
+                  }
+                }
+              EOS
+            end
+
   ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
   create_remote_file('site', '/etc/puppetlabs/code/environments/production/manifests/site.pp', site_pp)
   run_shell('chmod 644 /etc/puppetlabs/code/environments/production/manifests/site.pp')
@@ -133,8 +174,15 @@ def execute_agent(role)
 def reset_and_restart_containerd
   ['controller', 'worker1', 'worker2'].each do |node|
     ENV['TARGET_HOST'] = target_roles(node)[0][:name]
-    run_shell('rm -f /etc/containerd/config.toml')
-    run_shell('systemctl restart containerd')
+    if os_family.casecmp('redhat').zero?
+      run_shell('rm -f /etc/containerd/config.toml')
+      run_shell('systemctl restart containerd')
+    else
+      run_shell('wget /~https://github.com/containerd/containerd/releases/download/v1.6.12/containerd-1.6.12-linux-amd64.tar.gz && tar xvf containerd-1.6.12-linux-amd64.tar.gz')
+      run_shell('systemctl stop containerd')
+      run_shell('cd bin && cp * /usr/bin/')
+      run_shell('systemctl start containerd')
+    end
   end
 end
 
@@ -147,7 +195,11 @@ def open_communication_ports
       run_shell('iptables -I INPUT -p tcp -m multiport --dports 10251,10252,10255,30000:32767 -j ACCEPT')
     end
     run_shell('iptables -I INPUT -p udp -m multiport --dports 8472 -j ACCEPT')
-    run_shell('iptables-save > /etc/sysconfig/iptables')
+    if os_family.casecmp('redhat').zero?
+      run_shell('iptables-save > /etc/sysconfig/iptables')
+    else
+      run_shell('iptables-save > /etc/iptables/rules.v4')
+    end
   end
 end
 
@@ -157,6 +209,7 @@ def open_communication_ports
     hostname1, ipaddr1, int_ipaddr1 =  fetch_ip_hostname_by_role('controller')
     hostname2, ipaddr2, int_ipaddr2 =  fetch_ip_hostname_by_role('worker1')
     hostname3, ipaddr3, int_ipaddr3 =  fetch_ip_hostname_by_role('worker2')
+
     if c.filter.rules.key? :integration
       ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
       ['controller', 'worker1', 'worker2'].each do |node|
@@ -260,26 +313,48 @@ def open_communication_ports
     PUPPETCODE
 
     apply_manifest(pp)
-    if %r{debian|ubuntu-1604-lts}.match?(family)
+
+    if %r{debian|ubuntu}.match?(family)
       runtime = 'cri_containerd'
       cni = 'weave'
-      run_shell('apt-get update && apt-get install -y apt-transport-https')
-      run_shell('curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -')
-      run_shell('echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list')
-      run_shell('apt-get update')
-      run_shell('apt-get install -y kubectl')
-      run_shell('sudo apt install docker-ce=18.06.0~ce~3-0~ubuntu  docker-ce-cli=18.06.0~ce~3-0~ubuntu -y')
-      run_shell('sudo apt install docker.io -y')
-      run_shell('systemctl start docker.service')
-      run_shell('systemctl enable docker.service')
-      if family.include?('ubuntu-1604-lts')
-        run_shell('sudo ufw disable')
-      else
-        # Workaround for debian as the strech repositories do not have updated kubernetes packages
-        run_shell('echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" >> /etc/apt/sources.list.d/kube-xenial.list')
-        run_shell('/sbin/iptables -F')
+      ['controller', 'worker1', 'worker2'].each do |node|
+        ENV['TARGET_HOST'] = target_roles(node)[0][:name]
+
+        run_shell('apt update && apt install apt-transport-https -y')
+        run_shell('mkdir -p /etc/apt/keyrings')
+        run_shell('apt-get install -y curl gnupg2 software-properties-common')
+
+        if %r{debian-(10|11)}.match?(family)
+          run_shell('curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -')
+          run_shell('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"')
+        else
+          run_shell('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
+          run_shell('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"')
+        end
+        run_shell('curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg')
+        run_shell('echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list')
+
+        run_shell('apt-get update')
+        run_shell('apt install containerd -y')
+        run_shell('apt install kubectl kubelet kubeadm -y')
+        run_shell('apt-mark hold kubectl kubelet kubeadm')
+        run_shell('apt install docker-ce docker-ce-cli -y')
+        run_shell('apt install docker.io -y')
+        run_shell('systemctl start docker.service')
+        run_shell('systemctl enable docker.service')
+        run_shell('echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections')
+        run_shell('echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections')
+        run_shell('apt install iptables-persistent -y')
+        if family.include?('ubuntu')
+          run_shell('sudo ufw disable')
+        else
+          # Workaround for debian as the strech repositories do not have updated kubernetes packages
+          # run_shell('echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" >> /etc/apt/sources.list.d/kube-xenial.list')
+          run_shell('/sbin/iptables -F')
+        end
       end
     end
+
     if %r{rhel|redhat|centos}.match?(family)
       runtime = 'docker'
       cni = 'weave'
@@ -302,7 +377,8 @@ def open_communication_ports
     end
 
     ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
-    run_shell('docker build -t kubetool:latest /etc/puppetlabs/code/environments/production/modules/kubernetes/tooling')
+
+    run_shell('docker build -t kubetool:latest --network host /etc/puppetlabs/code/environments/production/modules/kubernetes/tooling')
 
     docker_run = <<~DOCKER
       docker run --rm -v $(pwd)/hieradata:/mnt -e OS=#{family} \
@@ -317,6 +393,7 @@ def open_communication_ports
     DOCKER
 
     run_shell(docker_run)
+
     create_remote_file('nginx', '/tmp/nginx.yml', nginx)
     create_remote_file('hiera', '/etc/puppetlabs/puppet/hiera.yaml', hiera)
     run_shell('chmod 644 /etc/puppetlabs/puppet/hiera.yaml')