Skip to content

Commit

Permalink
Extend cluster autoscaler to check AUTOSCALER_ENV_VARS in kube-env
Browse files Browse the repository at this point in the history 
This is the second part of the fix in kubernetes/kubernetes#61119

This provides a temporary way for the cluster autoscaler to get at
values that were removed from kube-env in kubernetes/kubernetes#60020.
Ideally this information will eventually be available via e.g. the Cluster API,
because kube-env is an internal interface that carries no stability
guarantees.
  • Loading branch information
@mtaufen
mtaufen committed Mar 16, 2018
1 parent 322d2e0 commit ee890f5
Show file tree
Hide file tree
Showing 2 changed files with 238 additions and 56 deletions.
79 changes: 66 additions & 13 deletions cluster-autoscaler/cloudprovider/gce/templates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,17 +332,31 @@ func parseKubeReserved(kubeReserved string) (apiv1.ResourceList, error) {
}

func extractLabelsFromKubeEnv(kubeEnv string) (map[string]string, error) {
labels, err := extractFromKubeEnv(kubeEnv, "NODE_LABELS")
// In v1.10+, labels are only exposed for the autoscaler via AUTOSCALER_ENV_VARS
// see kubernetes/kubernetes#61119. We try AUTOSCALER_ENV_VARS first, then
// fall back to the old way.
labels, err := extractAutoscalerVarFromKubeEnv(kubeEnv, "node_labels")
if err != nil {
return nil, err
glog.Errorf("node_labels not found via AUTOSCALER_ENV_VARS due to error, will try NODE_LABELS: %v", err)
labels, err = extractFromKubeEnv(kubeEnv, "NODE_LABELS")
if err != nil {
return nil, err
}
}
return parseKeyValueListToMap(labels)
}

func extractTaintsFromKubeEnv(kubeEnv string) ([]apiv1.Taint, error) {
taints, err := extractFromKubeEnv(kubeEnv, "NODE_TAINTS")
// In v1.10+, taints are only exposed for the autoscaler via AUTOSCALER_ENV_VARS
// see kubernetes/kubernetes#61119. We try AUTOSCALER_ENV_VARS first, then
// fall back to the old way.
taints, err := extractAutoscalerVarFromKubeEnv(kubeEnv, "node_taints")
if err != nil {
return nil, err
glog.Errorf("node_taints not found via AUTOSCALER_ENV_VARS due to error, will try NODE_TAINTS: %v", err)
taints, err = extractFromKubeEnv(kubeEnv, "NODE_TAINTS")
if err != nil {
return nil, err
}
}
taintMap, err := parseKeyValueListToMap(taints)
if err != nil {
Expand All @@ -352,19 +366,55 @@ func extractTaintsFromKubeEnv(kubeEnv string) ([]apiv1.Taint, error) {
}

func extractKubeReservedFromKubeEnv(kubeEnv string) (string, error) {
kubeletArgs, err := extractFromKubeEnv(kubeEnv, "KUBELET_TEST_ARGS")
// In v1.10+, kube-reserved is only exposed for the autoscaler via AUTOSCALER_ENV_VARS
// see kubernetes/kubernetes#61119. We try AUTOSCALER_ENV_VARS first, then
// fall back to the old way.
kubeReserved, err := extractAutoscalerVarFromKubeEnv(kubeEnv, "kube_reserved")
if err != nil {
return "", err
glog.Errorf("kube_reserved not found via AUTOSCALER_ENV_VARS due to error, will try kube-reserved in KUBELET_TEST_ARGS: %v", err)
kubeletArgs, err := extractFromKubeEnv(kubeEnv, "KUBELET_TEST_ARGS")
if err != nil {
return "", err
}
resourcesRegexp := regexp.MustCompile(`--kube-reserved=([^ ]+)`)

for _, value := range kubeletArgs {
matches := resourcesRegexp.FindStringSubmatch(value)
if len(matches) > 1 {
return matches[1], nil
}
}
return "", fmt.Errorf("kube-reserved not in kubelet args in kube-env: %q", strings.Join(kubeletArgs, " "))
}
resourcesRegexp := regexp.MustCompile(`--kube-reserved=([^ ]+)`)
return kubeReserved[0], nil
}

for _, value := range kubeletArgs {
matches := resourcesRegexp.FindStringSubmatch(value)
if len(matches) > 1 {
return matches[1], nil
func extractAutoscalerVarFromKubeEnv(kubeEnv, name string) ([]string, error) {
const autoscalerVars = "AUTOSCALER_ENV_VARS"
autoscalerVals, err := extractFromKubeEnv(kubeEnv, autoscalerVars)
if err != nil {
return nil, err
}
var result []string
for _, val := range autoscalerVals {
for _, v := range strings.Split(val, ";") {
v = strings.Trim(v, " ")
if len(v) == 0 {
continue
}
items := strings.SplitN(v, "=", 2)
if len(items) != 2 {
return nil, fmt.Errorf("malformed autoscaler var: %s", v)
}
if strings.Trim(items[0], " ") == name {
result = append(result, strings.Trim(items[1], " \"'"))
}
}
}
return "", fmt.Errorf("kube-reserved not in kubelet args in kube-env: %q", strings.Join(kubeletArgs, " "))
if len(result) == 0 {
return nil, fmt.Errorf("var %s not found in %s: %v", name, autoscalerVars, autoscalerVals)
}
return result, nil
}

func extractFromKubeEnv(kubeEnv, resource string) ([]string, error) {
Expand All @@ -391,10 +441,13 @@ func extractFromKubeEnv(kubeEnv, resource string) ([]string, error) {
func parseKeyValueListToMap(values []string) (map[string]string, error) {
result := make(map[string]string)
for _, value := range values {
if len(value) == 0 {
continue
}
for _, val := range strings.Split(value, ",") {
valItems := strings.SplitN(val, "=", 2)
if len(valItems) != 2 {
return nil, fmt.Errorf("error while parsing kube env value: %s", val)
return nil, fmt.Errorf("error while parsing key-value list, val: %s", val)
}
result[valItems[0]] = valItems[1]
}
Expand Down
215 changes: 172 additions & 43 deletions cluster-autoscaler/cloudprovider/gce/templates_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,27 +296,97 @@ func TestBuildAllocatableFromCapacity(t *testing.T) {
}
}

func TestExtractAutoscalerVarFromKubeEnv(t *testing.T) {
cases := []struct {
desc string
name string
env string
expect []string
err error
}{
{
desc: "node_labels",
name: "node_labels",
env: "AUTOSCALER_ENV_VARS: node_labels=a=b,c=d;node_taints=a=b:c,d=e:f\n",
expect: []string{"a=b,c=d"},
},
{
desc: "node_taints",
name: "node_taints",
env: "AUTOSCALER_ENV_VARS: node_labels=a=b,c=d;node_taints=a=b:c,d=e:f\n",
expect: []string{"a=b:c,d=e:f"},
},
{
desc: "malformed node_labels",
name: "node_labels",
env: "AUTOSCALER_ENV_VARS: node_labels;node_taints=a=b:c,d=e:f\n",
err: fmt.Errorf("malformed autoscaler var: node_labels"),
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
vals, err := extractAutoscalerVarFromKubeEnv(c.env, c.name)
assert.Equal(t, c.err, err)
if err != nil {
return
}
assert.Equal(t, c.expect, vals)
})
}
}

func TestExtractLabelsFromKubeEnv(t *testing.T) {
kubeenv := "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n"
expectedLabels := map[string]string{
"a": "b",
"c": "d",
"cloud.google.com/gke-nodepool": "pool-3",
"cloud.google.com/gke-preemptible": "true",
}
cases := []struct {
desc string
env string
expect map[string]string
err error
}{
{
desc: "from NODE_LABELS",
env: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n",
expect: expectedLabels,
err: nil,
},
{
desc: "from AUTOSCALER_ENV_VARS.node_labels",
env: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"AUTOSCALER_ENV_VARS: node_labels=a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true;" +
"node_taints='dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c';" +
"kube_reserved=cpu=1000m,memory=300000Mi\n" +
"DNS_SERVER_IP: '10.0.0.10'\n",
expect: expectedLabels,
err: nil,
},
{
desc: "malformed key-value in AUTOSCALER_ENV_VARS.node_labels",
env: "AUTOSCALER_ENV_VARS: node_labels=ab,c=d\n",
err: fmt.Errorf("error while parsing key-value list, val: ab"),
},
}

labels, err := extractLabelsFromKubeEnv(kubeenv)
assert.Nil(t, err)
assert.Equal(t, 4, len(labels))
assert.Equal(t, "b", labels["a"])
assert.Equal(t, "d", labels["c"])
assert.Equal(t, "pool-3", labels["cloud.google.com/gke-nodepool"])
assert.Equal(t, "true", labels["cloud.google.com/gke-preemptible"])
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
labels, err := extractLabelsFromKubeEnv(c.env)
assert.Equal(t, c.err, err)
if c.err != nil {
return
}
assert.Equal(t, c.expect, labels)
})
}
}

func TestExtractTaintsFromKubeEnv(t *testing.T) {
kubeenv := "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n"

expectedTaints := []apiv1.Taint{
expectedTaints := makeTaintSet([]apiv1.Taint{
{
Key: "dedicated",
Value: "ml",
Expand All @@ -332,12 +402,56 @@ func TestExtractTaintsFromKubeEnv(t *testing.T) {
Value: "b",
Effect: apiv1.TaintEffect("c"),
},
})

cases := []struct {
desc string
env string
expect map[apiv1.Taint]bool
err error
}{
{
desc: "from NODE_TAINTS",
env: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expect: expectedTaints,
},
{
desc: "from AUTOSCALER_ENV_VARS.node_taints",
env: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"AUTOSCALER_ENV_VARS: node_labels=a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true;" +
"node_taints='dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c';" +
"kube_reserved=cpu=1000m,memory=300000Mi\n",
expect: expectedTaints,
},
{
desc: "from empty AUTOSCALER_ENV_VARS.node_taints",
env: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"AUTOSCALER_ENV_VARS: node_labels=a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true;" +
"node_taints=\n",
expect: makeTaintSet([]apiv1.Taint{}),
},
{
desc: "malformed key-value in AUTOSCALER_ENV_VARS.node_taints",
env: "AUTOSCALER_ENV_VARS: node_taints='dedicatedml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
err: fmt.Errorf("error while parsing key-value list, val: dedicatedml:NoSchedule"),
},
}

taints, err := extractTaintsFromKubeEnv(kubeenv)
assert.Nil(t, err)
assert.Equal(t, 3, len(taints))
assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints))
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
taints, err := extractTaintsFromKubeEnv(c.env)
assert.Equal(t, c.err, err)
if c.err != nil {
return
}
assert.Equal(t, c.expect, makeTaintSet(taints))
})
}

}

Expand All @@ -348,29 +462,44 @@ func TestExtractKubeReservedFromKubeEnv(t *testing.T) {
expectedErr bool
}

testCases := []testCase{{
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"KUBELET_TEST_ARGS: --experimental-allocatable-ignore-eviction --kube-reserved=cpu=1000m,memory=300000Mi\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expectedReserved: "cpu=1000m,memory=300000Mi",
expectedErr: false,
}, {
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"KUBELET_TEST_ARGS: --experimental-allocatable-ignore-eviction\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expectedReserved: "",
expectedErr: true,
}, {
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expectedReserved: "",
expectedErr: true}}
testCases := []testCase{
{
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"KUBELET_TEST_ARGS: --experimental-allocatable-ignore-eviction --kube-reserved=cpu=1000m,memory=300000Mi\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expectedReserved: "cpu=1000m,memory=300000Mi",
expectedErr: false,
},
{
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"AUTOSCALER_ENV_VARS: node_labels=a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true;" +
"node_taints='dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c';" +
"kube_reserved=cpu=1000m,memory=300000Mi\n" +
"KUBELET_TEST_ARGS: --experimental-allocatable-ignore-eviction\n",
expectedReserved: "cpu=1000m,memory=300000Mi",
expectedErr: false,
},
{
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"KUBELET_TEST_ARGS: --experimental-allocatable-ignore-eviction\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expectedReserved: "",
expectedErr: true,
},
{
kubeEnv: "ENABLE_NODE_PROBLEM_DETECTOR: 'daemonset'\n" +
"NODE_LABELS: a=b,c=d,cloud.google.com/gke-nodepool=pool-3,cloud.google.com/gke-preemptible=true\n" +
"DNS_SERVER_IP: '10.0.0.10'\n" +
"NODE_TAINTS: 'dedicated=ml:NoSchedule,test=dev:PreferNoSchedule,a=b:c'\n",
expectedReserved: "",
expectedErr: true,
},
}

for _, tc := range testCases {
reserved, err := extractKubeReservedFromKubeEnv(tc.kubeEnv)
Expand Down

0 comments on commit ee890f5

Please sign in to comment.