diff --git a/README.md b/README.md index ede3ae058a1..5fbbeb796ac 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,7 @@ The following settings are optional and allow you to further configure your clus * `settings.kubernetes.cluster-domain`: The DNS domain for this cluster, allowing all Kubernetes-run containers to search this domain before the host's search domains. Defaults to `cluster.local`. * `settings.kubernetes.container-log-max-files`: The maximum number of container log files that can be present for a container. * `settings.kubernetes.container-log-max-size`: The maximum size of container log file before it is rotated. +* `settings.kubernetes.cpu-cfs-quota-enforced`: Whether CPU CFS quotas are enforced. Defaults to `true`. * `settings.kubernetes.cpu-manager-policy`: Specifies the CPU manager policy. Possible values are `static` and `none`. Defaults to `none`. If you want to allow pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node, you can set this setting to `static`. You should reboot if you change this setting after startup - try `apiclient reboot`. * `settings.kubernetes.cpu-manager-policy-options`: Policy options to apply when `cpu-manager-policy` is set to `static`. Currently `full-pcpus-only` is the only option. @@ -476,47 +477,90 @@ The following settings are optional and allow you to further configure your clus * `settings.kubernetes.event-burst`: The maximum size of a burst of event creations. * `settings.kubernetes.event-qps`: The maximum event creations per second. * `settings.kubernetes.eviction-hard`: The signals and thresholds that trigger pod eviction. +* `settings.kubernetes.eviction-max-pod-grace-period`: Maximum grace period, in seconds, to wait for pod termination before soft eviction. Default is `0`. +* `settings.kubernetes.eviction-soft`: The signals and thresholds that trigger pod eviction with a provided grace period. +* `settings.kubernetes.eviction-soft-grace-period`: Delay for each signal to wait for pod termination before eviction. + Remember to quote signals (since they all contain ".") and to quote all values. - Example user data for setting up eviction hard: + Example user data for setting up eviction values: ```toml [settings.kubernetes.eviction-hard] "memory.available" = "15%" + + [settings.kubernetes.eviction-soft] + "memory.available" = "12%" + + [settings.kubernetes.eviction-soft-grace-period] + "memory.available" = "30s" + + [settings.kubernetes] + "eviction-max-pod-grace-period" = 40 ``` * `settings.kubernetes.image-gc-high-threshold-percent`: The percent of disk usage after which image garbage collection is always run, expressed as an integer from 0-100 inclusive. * `settings.kubernetes.image-gc-low-threshold-percent`: The percent of disk usage before which image garbage collection is never run, expressed as an integer from 0-100 inclusive. -Since v1.14.0 `image-gc-high-threshold-percent` and `image-gc-low-threshold-percent` can be represented as numbers. -For example: + Since v1.14.0 `image-gc-high-threshold-percent` and `image-gc-low-threshold-percent` can be represented as numbers. + For example: -```toml -[settings.kubernetes] -image-gc-high-threshold-percent = 85 -image-gc-low-threshold-percent = 80 -``` + ```toml + [settings.kubernetes] + image-gc-high-threshold-percent = 85 + image-gc-low-threshold-percent = 80 + ``` -For backward compatibility, both string and numeric representations are accepted since v1.14.0. -Prior to v1.14.0 these needed to be represented as strings, for example: + For backward compatibility, both string and numeric representations are accepted since v1.14.0. + Prior to v1.14.0 these needed to be represented as strings, for example: -```toml -[settings.kubernetes] -image-gc-high-threshold-percent = "85" -image-gc-low-threshold-percent = "80" -``` + ```toml + [settings.kubernetes] + image-gc-high-threshold-percent = "85" + image-gc-low-threshold-percent = "80" + ``` -If you downgrade from v1.14.0 to an earlier version, and you have these values set as numbers, they will be converted to strings on downgrade. + If you downgrade from v1.14.0 to an earlier version, and you have these values set as numbers, they will be converted to strings on downgrade. * `settings.kubernetes.kube-api-burst`: The burst to allow while talking with kubernetes. * `settings.kubernetes.kube-api-qps`: The QPS to use while talking with kubernetes apiserver. * `settings.kubernetes.log-level`: Adjust the logging verbosity of the `kubelet` process. The default log level is 2, with higher numbers enabling more verbose logging. +* `settings.kubernetes.memory-manager-policy`: The memory management policy to use: `None` (default) or `Static`. + Note, when using the `Static` policy you should also set `settings.kubernetes.memory-manager-reserved-memory` values. +* `settings.kubernetes.memory-manager-reserved-memory`: Used to set the total amount of reserved memory for a node. + These settings are used to configure memory manager policy when `settings.kubernetes.memory-manager-policy` is set to `Static`. + + `memory-manager-reserved-memory` is set per NUMA node. For example: + + ```toml + [settings.kubernetes] + "memory-manager-policy" = "Static" + + [settings.kubernetes.memory-manager-reserved-memory.0] + # Reserve a single 1GiB huge page along with 674MiB of memory + "enabled" = true + "memory" = "674Mi" + "hugepages-1Gi" = "1Gi" + + [settings.kubernetes.memory-manager-reserved-memory.1] + # Reserve 1,074 2MiB huge pages + "enabled" = true + "hugepages-2Mi" = "2148Mi" + ``` + + **Warning:** `memory-manager-reserved-memory` settings are an advanced configuration and requires a clear understanding of what you are setting. + Misconfiguration of reserved memory settings may cause the Kubernetes `kubelet` process to fail. + It can be very difficult to recover from configuration errors. + Use the memory reservation information from `kubectl describe node` and make sure you understand the Kubernetes documentation related to the [memory manager](https://kubernetes.io/docs/tasks/administer-cluster/memory-manager/) and how to [reserve compute resources for system daemons](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/). + * `settings.kubernetes.pod-pids-limit`: The maximum number of processes per pod. * `settings.kubernetes.provider-id`: This sets the unique ID of the instance that an external provider (i.e. cloudprovider) can use to identify a specific node. * `settings.kubernetes.registry-burst`: The maximum size of bursty pulls. * `settings.kubernetes.registry-qps`: The registry pull QPS. * `settings.kubernetes.server-tls-bootstrap`: Enables or disables server certificate bootstrap. When enabled, the kubelet will request a certificate from the certificates.k8s.io API. This requires an approver to approve the certificate signing requests (CSR). Defaults to `true`. +* `settings.kubernetes.shutdown-grace-period`: Delay the node should wait for pod termination before shutdown. Default is `0s`. +* `settings.kubernetes.shutdown-grace-period-for-critical-pods`: The portion of the shutdown delay that should be dedicated to critical pod shutdown. Default is `0s`. * `settings.kubernetes.standalone-mode`: Whether to run the kubelet in standalone mode, without connecting to an API server. Defaults to `false`. * `settings.kubernetes.system-reserved`: Resources reserved for system components. diff --git a/Release.toml b/Release.toml index b5223b2eeb8..79d70ad500d 100644 --- a/Release.toml +++ b/Release.toml @@ -204,5 +204,6 @@ version = "1.14.0" "(1.13.5, 1.14.0)" = [ "migrate_v1.14.0_kubernetes-gc-percent-type-change.lz4", "migrate_v1.14.0_kubelet-config-settings.lz4", + "migrate_v1.14.0_kubelet-prefix-config-settings.lz4", "migrate_v1.14.0_k8s-services-mode.lz4", ] diff --git a/packages/kubernetes-1.22/kubelet-config b/packages/kubernetes-1.22/kubelet-config index 736263131cd..38888690d32 100644 --- a/packages/kubernetes-1.22/kubelet-config +++ b/packages/kubernetes-1.22/kubelet-config @@ -41,6 +41,21 @@ evictionHard: {{@key}}: "{{this}}" {{/each}} {{/if}} +{{#if settings.kubernetes.eviction-soft}} +evictionSoft: + {{#each settings.kubernetes.eviction-soft}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-soft-grace-period}} +evictionSoftGracePeriod: + {{#each settings.kubernetes.eviction-soft-grace-period}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-max-pod-grace-period}} +evictionMaxPodGracePeriod: {{settings.kubernetes.eviction-max-pod-grace-period}} +{{/if}} {{#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{/if}} @@ -76,6 +91,7 @@ systemReserved: {{/each}} systemReservedCgroup: "/system" {{/if}} +cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} @@ -133,3 +149,32 @@ containerLogMaxSize: {{settings.kubernetes.container-log-max-size}} {{#if settings.kubernetes.container-log-max-files includeZero=true}} containerLogMaxFiles: {{settings.kubernetes.container-log-max-files}} {{/if}} +{{#if settings.kubernetes.shutdown-grace-period}} +shutdownGracePeriod: {{settings.kubernetes.shutdown-grace-period}} +{{/if}} +{{#if settings.kubernetes.shutdown-grace-period-for-critical-pods}} +shutdownGracePeriodCriticalPods: {{settings.kubernetes.shutdown-grace-period-for-critical-pods}} +{{/if}} +{{#if settings.kubernetes.memory-manager-reserved-memory}} +{{#if (any_enabled settings.kubernetes.memory-manager-reserved-memory)}} +{{#if settings.kubernetes.memory-manager-policy}} +memoryManagerPolicy: {{settings.kubernetes.memory-manager-policy}} +{{/if}} +reservedMemory: +{{#each settings.kubernetes.memory-manager-reserved-memory}} +{{#if this.enabled}} + - numaNode: {{@key}} + limits: +{{#if this.memory}} + memory: {{this.memory}} +{{/if}} +{{#if this.hugepages-1Gi}} + hugepages-1Gi: {{this.hugepages-1Gi}} +{{/if}} +{{#if this.hugepages-2Mi}} + hugepages-2Mi: {{this.hugepages-2Mi}} +{{/if}} +{{/if}} +{{/each}} +{{/if}} +{{/if}} diff --git a/packages/kubernetes-1.23/kubelet-config b/packages/kubernetes-1.23/kubelet-config index 991f22b48a4..c1033b35671 100644 --- a/packages/kubernetes-1.23/kubelet-config +++ b/packages/kubernetes-1.23/kubelet-config @@ -41,6 +41,21 @@ evictionHard: {{@key}}: "{{this}}" {{/each}} {{/if}} +{{#if settings.kubernetes.eviction-soft}} +evictionSoft: + {{#each settings.kubernetes.eviction-soft}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-soft-grace-period}} +evictionSoftGracePeriod: + {{#each settings.kubernetes.eviction-soft-grace-period}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-max-pod-grace-period}} +evictionMaxPodGracePeriod: {{settings.kubernetes.eviction-max-pod-grace-period}} +{{/if}} {{#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{/if}} @@ -76,6 +91,7 @@ systemReserved: {{/each}} systemReservedCgroup: "/system" {{/if}} +cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} @@ -135,3 +151,32 @@ containerLogMaxSize: {{settings.kubernetes.container-log-max-size}} {{#if settings.kubernetes.container-log-max-files includeZero=true}} containerLogMaxFiles: {{settings.kubernetes.container-log-max-files}} {{/if}} +{{#if settings.kubernetes.shutdown-grace-period}} +shutdownGracePeriod: {{settings.kubernetes.shutdown-grace-period}} +{{/if}} +{{#if settings.kubernetes.shutdown-grace-period-for-critical-pods}} +shutdownGracePeriodCriticalPods: {{settings.kubernetes.shutdown-grace-period-for-critical-pods}} +{{/if}} +{{#if settings.kubernetes.memory-manager-reserved-memory}} +{{#if (any_enabled settings.kubernetes.memory-manager-reserved-memory)}} +{{#if settings.kubernetes.memory-manager-policy}} +memoryManagerPolicy: {{settings.kubernetes.memory-manager-policy}} +{{/if}} +reservedMemory: +{{#each settings.kubernetes.memory-manager-reserved-memory}} +{{#if this.enabled}} + - numaNode: {{@key}} + limits: +{{#if this.memory}} + memory: {{this.memory}} +{{/if}} +{{#if this.hugepages-1Gi}} + hugepages-1Gi: {{this.hugepages-1Gi}} +{{/if}} +{{#if this.hugepages-2Mi}} + hugepages-2Mi: {{this.hugepages-2Mi}} +{{/if}} +{{/if}} +{{/each}} +{{/if}} +{{/if}} diff --git a/packages/kubernetes-1.24/kubelet-config b/packages/kubernetes-1.24/kubelet-config index d1e6871a1bb..d468e2bbc32 100644 --- a/packages/kubernetes-1.24/kubelet-config +++ b/packages/kubernetes-1.24/kubelet-config @@ -41,6 +41,21 @@ evictionHard: {{@key}}: "{{this}}" {{/each}} {{/if}} +{{#if settings.kubernetes.eviction-soft}} +evictionSoft: + {{#each settings.kubernetes.eviction-soft}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-soft-grace-period}} +evictionSoftGracePeriod: + {{#each settings.kubernetes.eviction-soft-grace-period}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-max-pod-grace-period}} +evictionMaxPodGracePeriod: {{settings.kubernetes.eviction-max-pod-grace-period}} +{{/if}} {{#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{/if}} @@ -76,6 +91,7 @@ systemReserved: {{/each}} systemReservedCgroup: "/system" {{/if}} +cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} @@ -134,3 +150,32 @@ containerLogMaxSize: {{settings.kubernetes.container-log-max-size}} {{#if settings.kubernetes.container-log-max-files includeZero=true}} containerLogMaxFiles: {{settings.kubernetes.container-log-max-files}} {{/if}} +{{#if settings.kubernetes.shutdown-grace-period}} +shutdownGracePeriod: {{settings.kubernetes.shutdown-grace-period}} +{{/if}} +{{#if settings.kubernetes.shutdown-grace-period-for-critical-pods}} +shutdownGracePeriodCriticalPods: {{settings.kubernetes.shutdown-grace-period-for-critical-pods}} +{{/if}} +{{#if settings.kubernetes.memory-manager-reserved-memory}} +{{#if (any_enabled settings.kubernetes.memory-manager-reserved-memory)}} +{{#if settings.kubernetes.memory-manager-policy}} +memoryManagerPolicy: {{settings.kubernetes.memory-manager-policy}} +{{/if}} +reservedMemory: +{{#each settings.kubernetes.memory-manager-reserved-memory}} +{{#if this.enabled}} + - numaNode: {{@key}} + limits: +{{#if this.memory}} + memory: {{this.memory}} +{{/if}} +{{#if this.hugepages-1Gi}} + hugepages-1Gi: {{this.hugepages-1Gi}} +{{/if}} +{{#if this.hugepages-2Mi}} + hugepages-2Mi: {{this.hugepages-2Mi}} +{{/if}} +{{/if}} +{{/each}} +{{/if}} +{{/if}} diff --git a/packages/kubernetes-1.25/kubelet-config b/packages/kubernetes-1.25/kubelet-config index d1e6871a1bb..d468e2bbc32 100644 --- a/packages/kubernetes-1.25/kubelet-config +++ b/packages/kubernetes-1.25/kubelet-config @@ -41,6 +41,21 @@ evictionHard: {{@key}}: "{{this}}" {{/each}} {{/if}} +{{#if settings.kubernetes.eviction-soft}} +evictionSoft: + {{#each settings.kubernetes.eviction-soft}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-soft-grace-period}} +evictionSoftGracePeriod: + {{#each settings.kubernetes.eviction-soft-grace-period}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-max-pod-grace-period}} +evictionMaxPodGracePeriod: {{settings.kubernetes.eviction-max-pod-grace-period}} +{{/if}} {{#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{/if}} @@ -76,6 +91,7 @@ systemReserved: {{/each}} systemReservedCgroup: "/system" {{/if}} +cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} @@ -134,3 +150,32 @@ containerLogMaxSize: {{settings.kubernetes.container-log-max-size}} {{#if settings.kubernetes.container-log-max-files includeZero=true}} containerLogMaxFiles: {{settings.kubernetes.container-log-max-files}} {{/if}} +{{#if settings.kubernetes.shutdown-grace-period}} +shutdownGracePeriod: {{settings.kubernetes.shutdown-grace-period}} +{{/if}} +{{#if settings.kubernetes.shutdown-grace-period-for-critical-pods}} +shutdownGracePeriodCriticalPods: {{settings.kubernetes.shutdown-grace-period-for-critical-pods}} +{{/if}} +{{#if settings.kubernetes.memory-manager-reserved-memory}} +{{#if (any_enabled settings.kubernetes.memory-manager-reserved-memory)}} +{{#if settings.kubernetes.memory-manager-policy}} +memoryManagerPolicy: {{settings.kubernetes.memory-manager-policy}} +{{/if}} +reservedMemory: +{{#each settings.kubernetes.memory-manager-reserved-memory}} +{{#if this.enabled}} + - numaNode: {{@key}} + limits: +{{#if this.memory}} + memory: {{this.memory}} +{{/if}} +{{#if this.hugepages-1Gi}} + hugepages-1Gi: {{this.hugepages-1Gi}} +{{/if}} +{{#if this.hugepages-2Mi}} + hugepages-2Mi: {{this.hugepages-2Mi}} +{{/if}} +{{/if}} +{{/each}} +{{/if}} +{{/if}} diff --git a/packages/kubernetes-1.26/kubelet-config b/packages/kubernetes-1.26/kubelet-config index d1e6871a1bb..d468e2bbc32 100644 --- a/packages/kubernetes-1.26/kubelet-config +++ b/packages/kubernetes-1.26/kubelet-config @@ -41,6 +41,21 @@ evictionHard: {{@key}}: "{{this}}" {{/each}} {{/if}} +{{#if settings.kubernetes.eviction-soft}} +evictionSoft: + {{#each settings.kubernetes.eviction-soft}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-soft-grace-period}} +evictionSoftGracePeriod: + {{#each settings.kubernetes.eviction-soft-grace-period}} + {{@key}}: "{{this}}" + {{/each}} +{{/if}} +{{#if settings.kubernetes.eviction-max-pod-grace-period}} +evictionMaxPodGracePeriod: {{settings.kubernetes.eviction-max-pod-grace-period}} +{{/if}} {{#if settings.kubernetes.allowed-unsafe-sysctls}} allowedUnsafeSysctls: {{settings.kubernetes.allowed-unsafe-sysctls}} {{/if}} @@ -76,6 +91,7 @@ systemReserved: {{/each}} systemReservedCgroup: "/system" {{/if}} +cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} @@ -134,3 +150,32 @@ containerLogMaxSize: {{settings.kubernetes.container-log-max-size}} {{#if settings.kubernetes.container-log-max-files includeZero=true}} containerLogMaxFiles: {{settings.kubernetes.container-log-max-files}} {{/if}} +{{#if settings.kubernetes.shutdown-grace-period}} +shutdownGracePeriod: {{settings.kubernetes.shutdown-grace-period}} +{{/if}} +{{#if settings.kubernetes.shutdown-grace-period-for-critical-pods}} +shutdownGracePeriodCriticalPods: {{settings.kubernetes.shutdown-grace-period-for-critical-pods}} +{{/if}} +{{#if settings.kubernetes.memory-manager-reserved-memory}} +{{#if (any_enabled settings.kubernetes.memory-manager-reserved-memory)}} +{{#if settings.kubernetes.memory-manager-policy}} +memoryManagerPolicy: {{settings.kubernetes.memory-manager-policy}} +{{/if}} +reservedMemory: +{{#each settings.kubernetes.memory-manager-reserved-memory}} +{{#if this.enabled}} + - numaNode: {{@key}} + limits: +{{#if this.memory}} + memory: {{this.memory}} +{{/if}} +{{#if this.hugepages-1Gi}} + hugepages-1Gi: {{this.hugepages-1Gi}} +{{/if}} +{{#if this.hugepages-2Mi}} + hugepages-2Mi: {{this.hugepages-2Mi}} +{{/if}} +{{/if}} +{{/each}} +{{/if}} +{{/if}} diff --git a/sources/Cargo.lock b/sources/Cargo.lock index f519a6a3888..95201474b92 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -2248,6 +2248,13 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "kubelet-prefix-config-settings" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "kubernetes-gc-percent-type-change" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index f369ae3b757..b16c7ff39b0 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -46,6 +46,7 @@ members = [ "api/migration/migrations/v1.13.4/add-hostname-override-metadata", "api/migration/migrations/v1.14.0/kubernetes-gc-percent-type-change", "api/migration/migrations/v1.14.0/kubelet-config-settings", + "api/migration/migrations/v1.14.0/kubelet-prefix-config-settings", "api/migration/migrations/v1.14.0/k8s-services-mode", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs index 06ba834ddc9..1cc0e0f932c 100644 --- a/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs +++ b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs @@ -5,6 +5,13 @@ use std::process; fn run() -> Result<()> { migrate(AddSettingsMigration(&[ "settings.kubernetes.cpu-manager-policy-options", + "settings.kubernetes.cpu-cfs-quota-enforced", + "settings.kubernetes.shutdown-grace-period", + "settings.kubernetes.shutdown-grace-period-for-critical-pods", + "settings.kubernetes.eviction-soft", + "settings.kubernetes.eviction-soft-grace-period", + "settings.kubernetes.eviction-max-pod-grace-period", + "settings.kubernetes.memory-manager-policy", ])) } diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-prefix-config-settings/Cargo.toml b/sources/api/migration/migrations/v1.14.0/kubelet-prefix-config-settings/Cargo.toml new file mode 100644 index 00000000000..66333862690 --- /dev/null +++ b/sources/api/migration/migrations/v1.14.0/kubelet-prefix-config-settings/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kubelet-prefix-config-settings" +version = "0.1.0" +authors = ["Sean McGinnis "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers", version = "0.1.0"} diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-prefix-config-settings/src/main.rs b/sources/api/migration/migrations/v1.14.0/kubelet-prefix-config-settings/src/main.rs new file mode 100644 index 00000000000..1e83c2b5a24 --- /dev/null +++ b/sources/api/migration/migrations/v1.14.0/kubelet-prefix-config-settings/src/main.rs @@ -0,0 +1,19 @@ +use migration_helpers::{common_migrations::AddPrefixesMigration, migrate, Result}; +use std::process; + +/// Additional `settings.kubernetes` options for this release. +fn run() -> Result<()> { + migrate(AddPrefixesMigration(vec![ + "settings.kubernetes.memory-manager-reserved-memory", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// /~https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index c712dc4d304..03568bb600b 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -173,6 +173,10 @@ mod variant; mod de; use modeled_types::KubernetesCPUManagerPolicyOption; +use modeled_types::KubernetesEvictionKey; +use modeled_types::KubernetesMemoryManagerPolicy; +use modeled_types::KubernetesMemoryReservation; +use modeled_types::NonNegativeInteger; pub use variant::*; // Types used to communicate between client and server for 'apiclient exec'. @@ -193,11 +197,11 @@ use crate::modeled_types::{ DNSDomain, ECSAgentImagePullBehavior, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, ECSDurationValue, EtcHostsEntries, FriendlyVersion, Identifier, IntegerPercent, KmodKey, KubernetesAuthenticationMode, KubernetesBootstrapToken, KubernetesCloudProvider, - KubernetesClusterDnsIp, KubernetesClusterName, KubernetesDurationValue, - KubernetesEvictionHardKey, KubernetesLabelKey, KubernetesLabelValue, KubernetesQuantityValue, - KubernetesReservedResourceKey, KubernetesTaintValue, KubernetesThresholdValue, Lockdown, - OciDefaultsCapability, OciDefaultsResourceLimitType, PemCertificateString, SingleLineString, - SysctlKey, TopologyManagerPolicy, TopologyManagerScope, Url, ValidBase64, ValidLinuxHostname, + KubernetesClusterDnsIp, KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey, + KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey, + KubernetesTaintValue, KubernetesThresholdValue, Lockdown, OciDefaultsCapability, + OciDefaultsResourceLimitType, PemCertificateString, SingleLineString, SysctlKey, + TopologyManagerPolicy, TopologyManagerScope, Url, ValidBase64, ValidLinuxHostname, }; // Kubernetes static pod manifest settings @@ -228,7 +232,10 @@ struct KubernetesSettings { authentication_mode: KubernetesAuthenticationMode, bootstrap_token: KubernetesBootstrapToken, standalone_mode: bool, - eviction_hard: HashMap, + eviction_hard: HashMap, + eviction_soft: HashMap, + eviction_soft_grace_period: HashMap, + eviction_max_pod_grace_period: NonNegativeInteger, kube_reserved: HashMap, system_reserved: HashMap, allowed_unsafe_sysctls: Vec, @@ -242,6 +249,7 @@ struct KubernetesSettings { kube_api_burst: i32, container_log_max_size: KubernetesQuantityValue, container_log_max_files: i32, + cpu_cfs_quota_enforced: bool, cpu_manager_policy: CpuManagerPolicy, cpu_manager_reconcile_period: KubernetesDurationValue, cpu_manager_policy_options: Vec, @@ -255,6 +263,10 @@ struct KubernetesSettings { credential_providers: HashMap, server_certificate: ValidBase64, server_key: ValidBase64, + shutdown_grace_period: KubernetesDurationValue, + shutdown_grace_period_for_critical_pods: KubernetesDurationValue, + memory_manager_reserved_memory: HashMap, + memory_manager_policy: KubernetesMemoryManagerPolicy, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not. diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs index 45b4b42ff53..f8876e41a00 100644 --- a/sources/models/src/modeled_types/kubernetes.rs +++ b/sources/models/src/modeled_types/kubernetes.rs @@ -464,17 +464,12 @@ mod test_kubernetes_bootstrap_token { // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -/// KubernetesEvictionHardKey represents a string that contains a valid Kubernetes eviction hard key. +/// KubernetesEvictionKey represents a string that contains a valid Kubernetes eviction key. /// https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/ -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct KubernetesEvictionHardKey { - inner: String, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Scalar)] #[serde(rename_all = "lowercase")] -enum EvictionSignal { +pub enum KubernetesEvictionKey { #[serde(rename = "memory.available")] MemoryAvailable, #[serde(rename = "nodefs.available")] @@ -489,27 +484,13 @@ enum EvictionSignal { PidAvailable, } -impl TryFrom<&str> for KubernetesEvictionHardKey { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - serde_plain::from_str::(input).context(error::InvalidPlainValueSnafu { - field: "Eviction Hard key", - })?; - Ok(KubernetesEvictionHardKey { - inner: input.to_string(), - }) - } -} -string_impls_for!(KubernetesEvictionHardKey, "KubernetesEvictionHardKey"); - #[cfg(test)] -mod test_kubernetes_eviction_hard_key { - use super::KubernetesEvictionHardKey; +mod test_kubernetes_eviction_key { + use super::KubernetesEvictionKey; use std::convert::TryFrom; #[test] - fn good_eviction_hard_key() { + fn good_eviction_key() { for ok in &[ "memory.available", "nodefs.available", @@ -518,14 +499,14 @@ mod test_kubernetes_eviction_hard_key { "imagefs.inodesFree", "pid.available", ] { - KubernetesEvictionHardKey::try_from(*ok).unwrap(); + KubernetesEvictionKey::try_from(*ok).unwrap(); } } #[test] - fn bad_eviction_hard_key() { + fn bad_eviction_key() { for err in &["", "storage.available", ".bad", "bad.", &"a".repeat(64)] { - KubernetesEvictionHardKey::try_from(*err).unwrap_err(); + KubernetesEvictionKey::try_from(*err).unwrap_err(); } } } @@ -797,7 +778,9 @@ pub struct CpuManagerPolicy { #[derive(Debug, Clone, Eq, PartialEq, Hash, Deserialize)] #[serde(rename_all = "lowercase")] enum ValidCpuManagerPolicy { + #[serde(alias = "Static")] Static, + #[serde(alias = "None")] None, } @@ -821,7 +804,7 @@ mod test_cpu_manager_policy { #[test] fn good_cpu_manager_policy() { - for ok in &["static", "none"] { + for ok in &["Static", "static", "None", "none"] { CpuManagerPolicy::try_from(*ok).unwrap(); } } @@ -1358,3 +1341,75 @@ mod test_kubernetes_cpu_manager_policy_option { } } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesMemoryReservationKey represents a string that contains a valid Kubernetes memory +/// resource reservation key. +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Scalar)] +pub enum KubernetesMemoryReservationKey { + #[serde(rename = "memory")] + Memory, + #[serde(rename = "hugepages-2Mi")] + HugePages2Mi, + #[serde(rename = "hugepages-1Gi")] + HugePages1Gi, +} + +#[cfg(test)] +mod test_memory_reservation_key { + use super::KubernetesMemoryReservationKey; + use std::convert::TryFrom; + + #[test] + fn good_memory_reservation_key() { + for ok in &["memory", "hugepages-2Mi", "hugepages-1Gi"] { + KubernetesMemoryReservationKey::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_memory_reservation_key() { + for err in &["", "cpu", "hugepages-1Mi", "HugePages_1Gi", &"a".repeat(64)] { + KubernetesMemoryReservationKey::try_from(*err).unwrap_err(); + } + } +} + +/// KubernetesMemoryReservation enables setting kubelet reserved memory values. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct KubernetesMemoryReservation { + enabled: bool, + #[serde(flatten)] + limits: HashMap, +} + +/// KubernetesMemoryManagerPolicy represents the valid options for the memory manager policy. +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Scalar)] +pub enum KubernetesMemoryManagerPolicy { + #[serde(alias = "static")] + Static, + #[serde(alias = "none")] + None, +} + +#[cfg(test)] +mod test_kubernetes_memory_manager_policy { + use super::KubernetesMemoryManagerPolicy; + use std::convert::TryFrom; + + #[test] + fn good_policy_key() { + for ok in &["Static", "static", "None", "none"] { + KubernetesMemoryManagerPolicy::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_policy_key() { + for err in &["", "dynamic", &"a".repeat(64)] { + KubernetesMemoryManagerPolicy::try_from(*err).unwrap_err(); + } + } +} diff --git a/sources/models/src/modeled_types/shared.rs b/sources/models/src/modeled_types/shared.rs index 6de53ae8143..698c9765155 100644 --- a/sources/models/src/modeled_types/shared.rs +++ b/sources/models/src/modeled_types/shared.rs @@ -1,6 +1,9 @@ use super::error; use lazy_static::lazy_static; use regex::Regex; +use scalar::traits::{Scalar, Validate}; +use scalar::ValidationError; +use scalar_derive::Scalar; use semver::Version; use serde::de::Error as _; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -1130,3 +1133,44 @@ mod test_valid_kmod_key { assert!(KmodKey::try_from(vec!["z"; KMOD_KEY_LENGTH + 1].join("")).is_err()); } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// Input value that needs to be a positive value, but should not be greater +/// than an i32::MAX. +#[derive(Debug, PartialEq, Scalar)] +pub struct NonNegativeInteger { + inner: i32, +} + +impl Validate for NonNegativeInteger { + fn validate>(input: I) -> Result { + let inner: i32 = input.into(); + if inner < 0 { + Err(ValidationError::new( + "number must be great than or equal to 0", + )) + } else { + Ok(Self { inner }) + } + } +} + +#[cfg(test)] +mod test_positive_integer { + use super::NonNegativeInteger; + use std::convert::TryFrom; + + #[test] + fn valid_positive_integer() { + assert!(NonNegativeInteger::try_from(0).is_ok()); + assert!(NonNegativeInteger::try_from(i32::MAX).is_ok()); + assert!(NonNegativeInteger::try_from(42).is_ok()); + } + + #[test] + fn invalid_positive_integer() { + assert!(NonNegativeInteger::try_from(i32::MIN).is_err()); + assert!(NonNegativeInteger::try_from(-1).is_err()); + } +}