kubelet: enable credential provider support

This adds the capability to use Kubernetes image credential providers to retrieve credentials to use when pulling images for container creation. Initially we will only support the ecr-credential-provider, but things are set up so we may add more providers in future updates. Signed-off-by: Sean McGinnis <stmcg@amazon.com>
bottlerocket-os · Nov 3, 2022 · c361847 · c361847
1 parent a5a4135
commit c361847
Show file tree

Hide file tree

Showing 19 changed files with 147 additions and 1 deletion.
diff --git a/packages/kubernetes-1.21/credential-provider-config-yaml b/packages/kubernetes-1.21/credential-provider-config-yaml
@@ -0,0 +1,25 @@
+apiVersion: kubelet.config.k8s.io/v1alpha1
+kind: CredentialProviderConfig
+providers:
+{{#if settings.kubernetes.credential-providers}}
+{{#each settings.kubernetes.credential-providers}}
+{{#if this.enabled}}
+  - name: {{@key}}
+    matchImages:
+{{#each this.image-patterns}}
+      - "{{this}}"
+{{/each}}
+    defaultCacheDuration: "{{default "12h" this.cache-duration}}"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+{{#if (eq @key "ecr-credential-provider")}}
+    env:
+      - name: HOME
+        value: /root
+{{#if settings.aws.profile}}
+      - name: AWS_PROFILE
+        value: {{settings.aws.profile}}
+{{/if}}
+{{/if}}
+{{/if}}
+{{/each}}
+{{/if}}
diff --git a/packages/kubernetes-1.21/kubelet-config b/packages/kubernetes-1.21/kubelet-config
@@ -111,6 +111,7 @@ runtimeRequestTimeout: 15m
 featureGates:
   RotateKubeletServerCertificate: true
   CSIMigration: false
+  KubeletCredentialProviders: true
 protectKernelDefaults: true
 serializeImagePulls: false
 {{#if (and (default "" settings.kubernetes.server-certificate) (default "" settings.kubernetes.server-key))}}

diff --git a/packages/kubernetes-1.21/kubelet-exec-start-conf b/packages/kubernetes-1.21/kubelet-exec-start-conf
@@ -17,6 +17,12 @@ ExecStart=/usr/bin/kubelet \
     --network-plugin cni \
     --root-dir /var/lib/kubelet \
     --cert-dir /var/lib/kubelet/pki \
+{{#if settings.kubernetes.credential-providers}}
+{{#if (any_enabled settings.kubernetes.credential-providers)}}
+    --image-credential-provider-bin-dir /usr/libexec/kubernetes/kubelet/plugins \
+    --image-credential-provider-config /etc/kubernetes/kubelet/credential-provider-config.yaml \
+{{/if}}
+{{/if}}
     --node-ip ${NODE_IP} \
     --node-labels "${NODE_LABELS}" \
     --register-with-taints "${NODE_TAINTS}" \

diff --git a/packages/kubernetes-1.21/kubernetes-1.21.spec b/packages/kubernetes-1.21/kubernetes-1.21.spec
@@ -38,6 +38,7 @@ Source10: prepare-var-lib-kubelet.service
 Source11: kubelet-server-crt
 Source12: kubelet-server-key
 Source13: etc-kubernetes-pki.mount
+Source14: credential-provider-config-yaml
 
 # ExecStartPre drop-ins
 Source20: prestart-pull-pause-ctr.conf
@@ -107,6 +108,7 @@ install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/kubelet-exec-start-conf
 install -m 0644 %{S:7} %{buildroot}%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
 install -m 0644 %{S:11} %{buildroot}%{_cross_templatedir}/kubelet-server-crt
 install -m 0644 %{S:12} %{buildroot}%{_cross_templatedir}/kubelet-server-key
+install -m 0644 %{S:14} %{buildroot}%{_cross_templatedir}/credential-provider-config-yaml
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
 install -p -m 0644 %{S:8} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
@@ -142,6 +144,7 @@ ln -rs \
 %{_cross_templatedir}/kubernetes-ca-crt
 %{_cross_templatedir}/kubelet-server-crt
 %{_cross_templatedir}/kubelet-server-key
+%{_cross_templatedir}/credential-provider-config-yaml
 %{_cross_tmpfilesdir}/kubernetes.conf
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes

diff --git a/packages/kubernetes-1.22/credential-provider-config-yaml b/packages/kubernetes-1.22/credential-provider-config-yaml
@@ -0,0 +1,25 @@
+apiVersion: kubelet.config.k8s.io/v1alpha1
+kind: CredentialProviderConfig
+providers:
+{{#if settings.kubernetes.credential-providers}}
+{{#each settings.kubernetes.credential-providers}}
+{{#if this.enabled}}
+  - name: {{@key}}
+    matchImages:
+{{#each this.image-patterns}}
+      - "{{this}}"
+{{/each}}
+    defaultCacheDuration: "{{default "12h" this.cache-duration}}"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+{{#if (eq @key "ecr-credential-provider")}}
+    env:
+      - name: HOME
+        value: /root
+{{#if settings.aws.profile}}
+      - name: AWS_PROFILE
+        value: {{settings.aws.profile}}
+{{/if}}
+{{/if}}
+{{/if}}
+{{/each}}
+{{/if}}
diff --git a/packages/kubernetes-1.22/kubelet-config b/packages/kubernetes-1.22/kubelet-config
@@ -107,6 +107,7 @@ runtimeRequestTimeout: 15m
 featureGates:
   RotateKubeletServerCertificate: true
   CSIMigration: false
+  KubeletCredentialProviders: true
 protectKernelDefaults: true
 serializeImagePulls: false
 {{#if (and (default "" settings.kubernetes.server-certificate) (default "" settings.kubernetes.server-key))}}

diff --git a/packages/kubernetes-1.22/kubelet-exec-start-conf b/packages/kubernetes-1.22/kubelet-exec-start-conf
@@ -17,6 +17,12 @@ ExecStart=/usr/bin/kubelet \
     --network-plugin cni \
     --root-dir /var/lib/kubelet \
     --cert-dir /var/lib/kubelet/pki \
+{{#if settings.kubernetes.credential-providers}}
+{{#if (any_enabled settings.kubernetes.credential-providers)}}
+    --image-credential-provider-bin-dir /usr/libexec/kubernetes/kubelet/plugins \
+    --image-credential-provider-config /etc/kubernetes/kubelet/credential-provider-config.yaml \
+{{/if}}
+{{/if}}
     --node-ip ${NODE_IP} \
     --node-labels "${NODE_LABELS}" \
     --register-with-taints "${NODE_TAINTS}" \

diff --git a/packages/kubernetes-1.22/kubernetes-1.22.spec b/packages/kubernetes-1.22/kubernetes-1.22.spec
@@ -38,6 +38,7 @@ Source10: prepare-var-lib-kubelet.service
 Source11: kubelet-server-crt
 Source12: kubelet-server-key
 Source13: etc-kubernetes-pki.mount
+Source14: credential-provider-config-yaml
 
 # ExecStartPre drop-ins
 Source20: prestart-pull-pause-ctr.conf
@@ -104,6 +105,7 @@ install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/kubelet-exec-start-conf
 install -m 0644 %{S:7} %{buildroot}%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
 install -m 0644 %{S:11} %{buildroot}%{_cross_templatedir}/kubelet-server-crt
 install -m 0644 %{S:12} %{buildroot}%{_cross_templatedir}/kubelet-server-key
+install -m 0644 %{S:14} %{buildroot}%{_cross_templatedir}/credential-provider-config-yaml
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
 install -p -m 0644 %{S:8} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
@@ -139,6 +141,7 @@ ln -rs \
 %{_cross_templatedir}/kubernetes-ca-crt
 %{_cross_templatedir}/kubelet-server-crt
 %{_cross_templatedir}/kubelet-server-key
+%{_cross_templatedir}/credential-provider-config-yaml
 %{_cross_tmpfilesdir}/kubernetes.conf
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes

diff --git a/packages/kubernetes-1.23/credential-provider-config-yaml b/packages/kubernetes-1.23/credential-provider-config-yaml
@@ -0,0 +1,25 @@
+apiVersion: kubelet.config.k8s.io/v1alpha1
+kind: CredentialProviderConfig
+providers:
+{{#if settings.kubernetes.credential-providers}}
+{{#each settings.kubernetes.credential-providers}}
+{{#if this.enabled}}
+  - name: {{@key}}
+    matchImages:
+{{#each this.image-patterns}}
+      - "{{this}}"
+{{/each}}
+    defaultCacheDuration: "{{default "12h" this.cache-duration}}"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+{{#if (eq @key "ecr-credential-provider")}}
+    env:
+      - name: HOME
+        value: /root
+{{#if settings.aws.profile}}
+      - name: AWS_PROFILE
+        value: {{settings.aws.profile}}
+{{/if}}
+{{/if}}
+{{/if}}
+{{/each}}
+{{/if}}
diff --git a/packages/kubernetes-1.23/kubelet-config b/packages/kubernetes-1.23/kubelet-config
@@ -109,6 +109,7 @@ featureGates:
   CSIMigration: true
   CSIMigrationAWS: true
   CSIMigrationvSphere: true
+  KubeletCredentialProviders: true
 protectKernelDefaults: true
 serializeImagePulls: false
 {{#if (and (default "" settings.kubernetes.server-certificate) (default "" settings.kubernetes.server-key))}}

diff --git a/packages/kubernetes-1.23/kubelet-exec-start-conf b/packages/kubernetes-1.23/kubelet-exec-start-conf
@@ -17,6 +17,12 @@ ExecStart=/usr/bin/kubelet \
     --network-plugin cni \
     --root-dir /var/lib/kubelet \
     --cert-dir /var/lib/kubelet/pki \
+{{#if settings.kubernetes.credential-providers}}
+{{#if (any_enabled settings.kubernetes.credential-providers)}}
+    --image-credential-provider-bin-dir /usr/libexec/kubernetes/kubelet/plugins \
+    --image-credential-provider-config /etc/kubernetes/kubelet/credential-provider-config.yaml \
+{{/if}}
+{{/if}}
     --node-ip ${NODE_IP} \
     --node-labels "${NODE_LABELS}" \
     --register-with-taints "${NODE_TAINTS}" \

diff --git a/packages/kubernetes-1.23/kubernetes-1.23.spec b/packages/kubernetes-1.23/kubernetes-1.23.spec
@@ -38,6 +38,7 @@ Source10: prepare-var-lib-kubelet.service
 Source11: kubelet-server-crt
 Source12: kubelet-server-key
 Source13: etc-kubernetes-pki.mount
+Source14: credential-provider-config-yaml
 
 # ExecStartPre drop-ins
 Source20: prestart-pull-pause-ctr.conf
@@ -105,6 +106,7 @@ install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/kubelet-exec-start-conf
 install -m 0644 %{S:7} %{buildroot}%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
 install -m 0644 %{S:11} %{buildroot}%{_cross_templatedir}/kubelet-server-crt
 install -m 0644 %{S:12} %{buildroot}%{_cross_templatedir}/kubelet-server-key
+install -m 0644 %{S:14} %{buildroot}%{_cross_templatedir}/credential-provider-config-yaml
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
 install -p -m 0644 %{S:8} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
@@ -141,6 +143,7 @@ ln -rs \
 %{_cross_templatedir}/kubernetes-ca-crt
 %{_cross_templatedir}/kubelet-server-crt
 %{_cross_templatedir}/kubelet-server-key
+%{_cross_templatedir}/credential-provider-config-yaml
 %{_cross_tmpfilesdir}/kubernetes.conf
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes

diff --git a/packages/kubernetes-1.24/credential-provider-config-yaml b/packages/kubernetes-1.24/credential-provider-config-yaml
@@ -0,0 +1,25 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: CredentialProviderConfig
+providers:
+{{#if settings.kubernetes.credential-providers}}
+{{#each settings.kubernetes.credential-providers}}
+{{#if this.enabled}}
+  - name: {{@key}}
+    matchImages:
+{{#each this.image-patterns}}
+      - "{{this}}"
+{{/each}}
+    defaultCacheDuration: "{{default "12h" this.cache-duration}}"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1
+{{#if (eq @key "ecr-credential-provider")}}
+    env:
+      - name: HOME
+        value: /root
+{{#if settings.aws.profile}}
+      - name: AWS_PROFILE
+        value: {{settings.aws.profile}}
+{{/if}}
+{{/if}}
+{{/if}}
+{{/each}}
+{{/if}}
diff --git a/packages/kubernetes-1.24/kubelet-exec-start-conf b/packages/kubernetes-1.24/kubelet-exec-start-conf
@@ -16,6 +16,12 @@ ExecStart=/usr/bin/kubelet \
     --containerd=/run/containerd/containerd.sock \
     --root-dir /var/lib/kubelet \
     --cert-dir /var/lib/kubelet/pki \
+{{#if settings.kubernetes.credential-providers}}
+{{#if (any_enabled settings.kubernetes.credential-providers)}}
+    --image-credential-provider-bin-dir /usr/libexec/kubernetes/kubelet/plugins \
+    --image-credential-provider-config /etc/kubernetes/kubelet/credential-provider-config.yaml \
+{{/if}}
+{{/if}}
     --node-ip ${NODE_IP} \
     --node-labels "${NODE_LABELS}" \
     --register-with-taints "${NODE_TAINTS}" \

diff --git a/packages/kubernetes-1.24/kubernetes-1.24.spec b/packages/kubernetes-1.24/kubernetes-1.24.spec
@@ -46,6 +46,7 @@ Source10: prepare-var-lib-kubelet.service
 Source11: kubelet-server-crt
 Source12: kubelet-server-key
 Source13: etc-kubernetes-pki.mount
+Source14: credential-provider-config-yaml
 
 # ExecStartPre drop-ins
 Source20: prestart-pull-pause-ctr.conf
@@ -113,6 +114,7 @@ install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/kubelet-exec-start-conf
 install -m 0644 %{S:7} %{buildroot}%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
 install -m 0644 %{S:11} %{buildroot}%{_cross_templatedir}/kubelet-server-crt
 install -m 0644 %{S:12} %{buildroot}%{_cross_templatedir}/kubelet-server-key
+install -m 0644 %{S:14} %{buildroot}%{_cross_templatedir}/credential-provider-config-yaml
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
 install -p -m 0644 %{S:8} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
@@ -149,6 +151,7 @@ ln -rs \
 %{_cross_templatedir}/kubernetes-ca-crt
 %{_cross_templatedir}/kubelet-server-crt
 %{_cross_templatedir}/kubelet-server-key
+%{_cross_templatedir}/credential-provider-config-yaml
 %{_cross_tmpfilesdir}/kubernetes.conf
 %{_cross_sysctldir}/90-kubelet.conf
 %dir %{_cross_libexecdir}/kubernetes

diff --git a/sources/api/migration/migrations/v1.11.0/credential-providers/src/main.rs b/sources/api/migration/migrations/v1.11.0/credential-providers/src/main.rs
@@ -10,6 +10,7 @@ use std::process;
 fn run() -> Result<()> {
     migrate(AddPrefixesMigration(vec![
         "settings.kubernetes.credential-providers",
+        "configuration-files.credential-provider-config-yaml",
     ]))
 }
 

diff --git a/sources/api/migration/migrations/v1.11.0/kubelet-new-config-files/Cargo.toml b/sources/api/migration/migrations/v1.11.0/kubelet-new-config-files/Cargo.toml
@@ -9,4 +9,4 @@ publish = false
 exclude = ["README.md"]
 
 [dependencies]
-migration-helpers = { path = "../../../migration-helpers", version = "0.1.0" }
+migration-helpers = { path = "../../../migration-helpers", version = "0.1.0" }
diff --git a/sources/api/migration/migrations/v1.11.0/kubelet-new-config-files/src/main.rs b/sources/api/migration/migrations/v1.11.0/kubelet-new-config-files/src/main.rs
@@ -27,6 +27,7 @@ fn run() -> Result<()> {
             "proxy-env",
             "kubelet-server-crt",
             "kubelet-server-key",
+            "credential-provider-config-yaml",
         ],
     }]))
 }

diff --git a/sources/models/shared-defaults/kubernetes-services.toml b/sources/models/shared-defaults/kubernetes-services.toml
@@ -9,6 +9,7 @@ configuration-files = [
   "proxy-env",
   "kubelet-server-crt",
   "kubelet-server-key",
+  "credential-provider-config-yaml",
 ]
 restart-commands = [
   "/usr/bin/systemctl try-restart kubelet.service"
@@ -46,6 +47,10 @@ template-path = "/usr/share/templates/kubelet-server-key"
 path = "/etc/systemd/system/kubelet.service.d/exec-start.conf"
 template-path = "/usr/share/templates/kubelet-exec-start-conf"
 
+[configuration-files.credential-provider-config-yaml]
+path = "/etc/kubernetes/kubelet/credential-provider-config.yaml"
+template-path = "/usr/share/templates/credential-provider-config-yaml"
+
 [services.static-pods]
 configuration-files = []
 restart-commands = ["/usr/bin/static-pods"]