From 2e303872e6a1408b3e9da4bcea8354d1ea53a842 Mon Sep 17 00:00:00 2001 From: tariqibrahim Date: Mon, 12 Nov 2018 11:48:18 -0800 Subject: [PATCH 1/4] consistent use of kubernetes image base --- pkg/acsengine/params_k8s.go | 26 +++++++++++++------------- pkg/acsengine/params_k8s_test.go | 1 + pkg/acsengine/params_test.go | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 550253b043..1c6b6a901c 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -24,11 +24,11 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params k8sVersion := orchestratorProfile.OrchestratorVersion k8sComponents := api.K8sComponentsByVersionMap[k8sVersion] kubernetesConfig := orchestratorProfile.KubernetesConfig + kubernetesImageBase := kubernetesConfig.KubernetesImageBase if kubernetesConfig != nil { - if helpers.IsTrueBoolPointer(kubernetesConfig.UseCloudControllerManager) { - kubernetesCcmSpec := kubernetesConfig.KubernetesImageBase + k8sComponents["ccm"] + kubernetesCcmSpec := kubernetesImageBase + k8sComponents["ccm"] if kubernetesConfig.CustomCcmImage != "" { kubernetesCcmSpec = kubernetesConfig.CustomCcmImage } @@ -36,20 +36,20 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec) } - kubernetesHyperkubeSpec := kubernetesConfig.KubernetesImageBase + k8sComponents["hyperkube"] + kubernetesHyperkubeSpec := kubernetesImageBase + k8sComponents["hyperkube"] if kubernetesConfig.CustomHyperkubeImage != "" { kubernetesHyperkubeSpec = kubernetesConfig.CustomHyperkubeImage } addValue(parametersMap, "kubeDNSServiceIP", kubernetesConfig.DNSServiceIP) addValue(parametersMap, "kubernetesHyperkubeSpec", kubernetesHyperkubeSpec) - addValue(parametersMap, "kubernetesAddonManagerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["addonmanager"]) - addValue(parametersMap, "kubernetesAddonResizerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["addonresizer"]) + addValue(parametersMap, "kubernetesAddonManagerSpec", kubernetesImageBase+k8sComponents["addonmanager"]) + addValue(parametersMap, "kubernetesAddonResizerSpec", kubernetesImageBase+k8sComponents["addonresizer"]) if orchestratorProfile.NeedsExecHealthz() { - addValue(parametersMap, "kubernetesExecHealthzSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["exechealthz"]) + addValue(parametersMap, "kubernetesExecHealthzSpec", kubernetesImageBase+k8sComponents["exechealthz"]) } - addValue(parametersMap, "kubernetesDNSSidecarSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["k8s-dns-sidecar"]) - addValue(parametersMap, "kubernetesHeapsterSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["heapster"]) + addValue(parametersMap, "kubernetesDNSSidecarSpec", kubernetesImageBase+k8sComponents["k8s-dns-sidecar"]) + addValue(parametersMap, "kubernetesHeapsterSpec", kubernetesImageBase+k8sComponents["heapster"]) if kubernetesConfig.IsTillerEnabled() { tillerAddon := kubernetesConfig.GetAddonByName(DefaultTillerAddonName) tillerIndex := tillerAddon.GetAddonContainersIndexByName(DefaultTillerAddonName) @@ -163,7 +163,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params if metricsServerAddon.Containers[metricsServerIndex].Image != "" { addValue(parametersMap, "kubernetesMetricsServerSpec", metricsServerAddon.Containers[metricsServerIndex].Image) } else { - addValue(parametersMap, "kubernetesMetricsServerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents[DefaultMetricsServerAddonName]) + addValue(parametersMap, "kubernetesMetricsServerSpec", kubernetesImageBase+k8sComponents[DefaultMetricsServerAddonName]) } } } @@ -217,12 +217,12 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params } } if common.IsKubernetesVersionGe(k8sVersion, "1.12.0") { - addValue(parametersMap, "kubernetesCoreDNSSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["coredns"]) + addValue(parametersMap, "kubernetesCoreDNSSpec", kubernetesImageBase+k8sComponents["coredns"]) } else { - addValue(parametersMap, "kubernetesKubeDNSSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["kube-dns"]) - addValue(parametersMap, "kubernetesDNSMasqSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["dnsmasq"]) + addValue(parametersMap, "kubernetesKubeDNSSpec", kubernetesImageBase+k8sComponents["kube-dns"]) + addValue(parametersMap, "kubernetesDNSMasqSpec", kubernetesImageBase+k8sComponents["dnsmasq"]) } - addValue(parametersMap, "kubernetesPodInfraContainerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["pause"]) + addValue(parametersMap, "kubernetesPodInfraContainerSpec", kubernetesImageBase+k8sComponents["pause"]) addValue(parametersMap, "cloudproviderConfig", api.CloudProviderConfig{ CloudProviderBackoff: kubernetesConfig.CloudProviderBackoff, CloudProviderBackoffRetries: kubernetesConfig.CloudProviderBackoffRetries, diff --git a/pkg/acsengine/params_k8s_test.go b/pkg/acsengine/params_k8s_test.go index f76cca2c59..926956e6df 100644 --- a/pkg/acsengine/params_k8s_test.go +++ b/pkg/acsengine/params_k8s_test.go @@ -38,6 +38,7 @@ func TestAssignKubernetesParameters(t *testing.T) { parametersMap := paramsMap{} containerService.Location = "eatsus" cloudSpecConfig := containerService.GetCloudSpecConfig() + containerService.SetPropertiesDefaults(false, false) assignKubernetesParameters(containerService.Properties, parametersMap, cloudSpecConfig, DefaultGeneratorCode) for k, v := range parametersMap { switch val := v.(paramsMap)["value"].(type) { diff --git a/pkg/acsengine/params_test.go b/pkg/acsengine/params_test.go index a045ee80ad..20e8c3e2d2 100644 --- a/pkg/acsengine/params_test.go +++ b/pkg/acsengine/params_test.go @@ -34,6 +34,7 @@ func TestAssignParameters(t *testing.T) { } containerService.Location = "eastus" + containerService.SetPropertiesDefaults(false, false) parametersMap, err := getParameters(containerService, DefaultGeneratorCode, "testversion") if err != nil { t.Errorf("should not get error when populating parameters") From 5f3558e80bdd71cf545a3dc6fc675a50c8f5526e Mon Sep 17 00:00:00 2001 From: tariqibrahim Date: Wed, 14 Nov 2018 11:27:03 -0800 Subject: [PATCH 2/4] resolve merge conflicts --- pkg/acsengine/params_k8s.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 67549197be..5bbf88a652 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -48,8 +48,8 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params if orchestratorProfile.NeedsExecHealthz() { addValue(parametersMap, "kubernetesExecHealthzSpec", kubernetesImageBase+k8sComponents["exechealthz"]) } - addValue(parametersMap, "kubernetesDNSSidecarSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["k8s-dns-sidecar"]) - addValue(parametersMap, "kubernetesHeapsterSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["heapster"]) + addValue(parametersMap, "kubernetesDNSSidecarSpec", kubernetesImageBase+k8sComponents["k8s-dns-sidecar"]) + addValue(parametersMap, "kubernetesHeapsterSpec", kubernetesImageBase+k8sComponents["heapster"]) if kubernetesConfig.IsAADPodIdentityEnabled() { aadPodIdentityAddon := kubernetesConfig.GetAddonByName(DefaultAADPodIdentityAddonName) aadIndex := aadPodIdentityAddon.GetAddonContainersIndexByName(DefaultAADPodIdentityAddonName) From 443bbcdb1c29d0d9b53a7d4963079ecfcd2aa8ee Mon Sep 17 00:00:00 2001 From: tariqibrahim Date: Wed, 14 Nov 2018 12:14:17 -0800 Subject: [PATCH 3/4] fix review comments --- pkg/api/defaults-kubelet.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/api/defaults-kubelet.go b/pkg/api/defaults-kubelet.go index b01d79773c..df2857c995 100644 --- a/pkg/api/defaults-kubelet.go +++ b/pkg/api/defaults-kubelet.go @@ -10,7 +10,6 @@ import ( func (cs *ContainerService) setKubeletConfig() { o := cs.Properties.OrchestratorProfile - cloudSpecConfig := cs.GetCloudSpecConfig() staticLinuxKubeletConfig := map[string]string{ "--address": "0.0.0.0", "--allow-privileged": "true", @@ -48,7 +47,7 @@ func (cs *ContainerService) setKubeletConfig() { defaultKubeletConfig := map[string]string{ "--cluster-domain": "cluster.local", "--network-plugin": "cni", - "--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + K8sComponentsByVersionMap[o.OrchestratorVersion]["pause"], + "--pod-infra-container-image": o.KubernetesConfig.KubernetesImageBase + K8sComponentsByVersionMap[o.OrchestratorVersion]["pause"], "--max-pods": strconv.Itoa(DefaultKubernetesMaxPods), "--eviction-hard": DefaultKubernetesHardEvictionThreshold, "--node-status-update-frequency": K8sComponentsByVersionMap[o.OrchestratorVersion]["nodestatusfreq"], From de67ece149a2c34c6980a9bcc6c1e8b94f64a19a Mon Sep 17 00:00:00 2001 From: tariqibrahim Date: Wed, 14 Nov 2018 12:55:54 -0800 Subject: [PATCH 4/4] update docs --- docs/clusterdefinition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 119416740e..5944f9878c 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -59,7 +59,7 @@ To learn more about supported orchestrators and versions, run the orchestrators | gcHighThreshold | no | Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) | | gcLowThreshold | no | Sets the --image-gc-low-threshold value on the kublet configuration. Default is 80. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) | | kubeletConfig | no | Configure various runtime configuration for kubelet. See `kubeletConfig` [below](#feat-kubelet-config) | -| kubernetesImageBase | no | Specifies the base URL (everything preceding the actual image filename) of the kubernetes hyperkube image to use for cluster deployment, e.g., `k8s.gcr.io/` | +| kubernetesImageBase | no | Specifies the default image base URL (everything preceding the actual image filename) to be used for all kubernetes-related containers such as hyperkube, cloud-controller-manager, pause, addon-manager, heapster, exechealthz etc. e.g., `k8s.gcr.io/` | | loadBalancerSku | no | Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`. If not set, it will be default to basic. Requires Kubernetes 1.11 or newer. NOTE: VMs behind ILB standard SKU will not be able to access the internet without ELB configured with at least one frontend IP as described in the [standard loadbalancer outbound connectivity doc](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-overview#control-outbound-connectivity). For Kubernetes 1.11 and 1.12, We have created an external loadbalancer service in the kube-system namespace as a workaround to this issue. Starting k8s 1.13, instead of creating an ELB service, we will setup outbound rules in ARM template once the API is available. | | networkPlugin | no | Specifies the network plugin implementation for the cluster. Valid values are:
`"azure"` (default), which provides an Azure native networking experience
`"kubenet"` for k8s software networking implementation.
`"flannel"` for using CoreOS Flannel
`"cilium"` for using the default Cilium CNI IPAM | | networkPolicy | no | Specifies the network policy enforcement tool for the cluster (currently Linux-only). Valid values are:
`"calico"` for Calico network policy.
`"cilium"` for cilium network policy (Lin), and `"azure"` (experimental) for Azure CNI-compliant network policy (note: Azure CNI-compliant network policy requires explicit `"networkPlugin": "azure"` configuration as well).
See [network policy examples](../examples/networkpolicy) for more information. |