From f6f53f3cc5a561a09c1e0b3c695b9dc88aecfb5d Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 27 Sep 2018 15:09:15 -0700 Subject: [PATCH 01/15] ip-masq-agent as addon --- parts/k8s/addons/ip-masq-agent.yaml | 55 ++++++++++++++++++++ parts/k8s/kubernetesagentcustomdata.yml | 4 -- parts/k8s/kubernetesmastercustomdata.yml | 7 ++- parts/k8s/kubernetesmastercustomdatavmss.yml | 1 - parts/k8s/kubernetesparams.t | 24 +++++++++ pkg/acsengine/addons.go | 14 +++++ pkg/acsengine/artifacts.go | 9 +++- pkg/acsengine/const.go | 2 + pkg/acsengine/defaults-kubelet.go | 2 +- pkg/api/const.go | 2 + pkg/api/types.go | 5 ++ pkg/api/types_test.go | 31 +++++++++++ 12 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 parts/k8s/addons/ip-masq-agent.yaml diff --git a/parts/k8s/addons/ip-masq-agent.yaml b/parts/k8s/addons/ip-masq-agent.yaml new file mode 100644 index 0000000000..fb2a7b424f --- /dev/null +++ b/parts/k8s/addons/ip-masq-agent.yaml @@ -0,0 +1,55 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: azure-ip-masq-agent + namespace: kube-system + labels: + component: azure-ip-masq-agent + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + tier: node +spec: + template: + metadata: + labels: + k8s-app: azure-ip-masq-agent + tier: node + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + containers: + - name: azure-ip-masq-agent + image: gcr.io/google-containers/ip-masq-agent-amd64:v2.0.0 + securityContext: + privileged: true + volumeMounts: + - name: azure-ip-masq-agent-config-volume + mountPath: /etc/config + resources: + requests: + cpu: + memory: + limits: + cpu: + memory: + volumes: + - name: azure-ip-masq-agent-config-volume + configMap: + name: azure-ip-masq-agent-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: azure-ip-masq-agent-config + namespace: kube-system + labels: + component: azure-ip-masq-agent + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: EnsureExists +data: + ip-masq-agent: |- + nonMasqueradeCIDRs: + + masqLinkLocal: true + resyncInterval: 60s \ No newline at end of file diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 56e1385eb6..6af747e9aa 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -188,10 +188,6 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER owner: "root" content: | #!/bin/bash -{{if IsAzureCNI}} - # SNAT outbound traffic from pods to destinations outside of VNET. - iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE -{{end}} {{if not EnablePodSecurityPolicy}} sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service" {{end}} diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 819c8585a2..f6de0e9d0c 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -242,7 +242,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{if IsAzureCNI}} # SNAT outbound traffic from pods to destinations outside of VNET. - iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE sed -i "s||{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml" {{end}} sed -i "s||{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" @@ -393,6 +392,12 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" {{end}} + sed -i "s||{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + - path: "/opt/azure/containers/provision.sh" permissions: "0744" encoding: gzip diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml index ac42acb521..b43b7ed392 100644 --- a/parts/k8s/kubernetesmastercustomdatavmss.yml +++ b/parts/k8s/kubernetesmastercustomdatavmss.yml @@ -244,7 +244,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{if IsAzureCNI}} # SNAT outbound traffic from pods to destinations outside of VNET. - iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE sed -i "s||{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml" {{end}} sed -i "s||{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 70616d3d4d..4bfcdebe4c 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -577,6 +577,30 @@ "type": "string" }, {{end}} + "kubernetesIPMasqAgentCPURequests": { + "metadata": { + "description": "IP Masq Agent CPU Requests" + }, + "type": "string" + }, + "kubernetesIPMasqAgentMemoryRequests": { + "metadata": { + "description": "IP Masq Agent Memory Requests" + }, + "type": "string" + }, + "kubernetesIPMasqAgentCPULimit": { + "metadata": { + "description": "IP Masq Agent CPU Limit" + }, + "type": "string" + }, + "kubernetesIPMasqAgentMemoryLimit": { + "metadata": { + "description": "IP Masq Agent Memory Limit" + }, + "type": "string" + }, "kubernetesPodInfraContainerSpec": { "metadata": { "description": "The container spec for pod infra." diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index d7e94e7190..40b6d2c24f 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -179,6 +179,19 @@ func setAddonsConfig(cs *api.ContainerService) { }, } + defaultIPMasqAgentAddonsConfig := api.KubernetesAddon{ + Name: IPMASQAgentAddonName, + Enabled: helpers.PointerToBool(true), + Containers: []api.KubernetesContainerSpec{ + { + CPURequests: "50m", + MemoryRequests: "10Mi", + CPULimits: "50m", + MemoryLimits: "10Mi", + }, + }, + } + defaultAzureCNINetworkMonitorAddonsConfig := api.KubernetesAddon{ Name: AzureCNINetworkMonitoringAddonName, Enabled: azureCNINetworkMonitorAddonEnabled(o), @@ -213,6 +226,7 @@ func setAddonsConfig(cs *api.ContainerService) { defaultContainerMonitoringAddonsConfig, defaultAzureCNINetworkMonitorAddonsConfig, defaultAzureNetworkPolicyAddonsConfig, + defaultIPMasqAgentAddonsConfig, } // Add default addons specification, if no user-provided spec exists if o.KubernetesConfig.Addons == nil { diff --git a/pkg/acsengine/artifacts.go b/pkg/acsengine/artifacts.go index 743890f97f..5172d5e997 100644 --- a/pkg/acsengine/artifacts.go +++ b/pkg/acsengine/artifacts.go @@ -120,7 +120,6 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultReschedulerAddonName), }, { - kubernetesFeatureSetting{ "kubernetesmasteraddons-azure-npm-daemonset.yaml", "azure-npm-daemonset.yaml", @@ -233,6 +232,14 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti }, profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultELBSVCAddonName), }, + { + kubernetesFeatureSetting{ + "ip-masq-agent.yaml", + "ip-masq-agent.yaml", + true, + }, + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(IPMASQAgentAddonName), + }, } } diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 31aeb77ce0..972f00ad4a 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -160,6 +160,8 @@ const ( AzureCNINetworkMonitoringAddonName = "azure-cni-networkmonitor" // AzureNetworkPolicyAddonName is the name of the Azure CNI networkmonitor addon AzureNetworkPolicyAddonName = "azure-npm-daemonset" + // IPMASQAgentAddonName is the name of the ip masq agent addon + IPMASQAgentAddonName = "ip-masq-agent" // DefaultKubernetesKubeletMaxPods is the max pods per kubelet DefaultKubernetesKubeletMaxPods = 110 // DefaultMasterEtcdServerPort is the default etcd server port for Kubernetes master nodes diff --git a/pkg/acsengine/defaults-kubelet.go b/pkg/acsengine/defaults-kubelet.go index dcf73b1cbc..5159fefafe 100644 --- a/pkg/acsengine/defaults-kubelet.go +++ b/pkg/acsengine/defaults-kubelet.go @@ -55,7 +55,7 @@ func setKubeletConfig(cs *api.ContainerService) { "--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"], "--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), - "--non-masquerade-cidr": o.KubernetesConfig.ClusterSubnet, + "--non-masquerade-cidr": "0.0.0.0", "--cloud-provider": "azure", "--cloud-config": "/etc/kubernetes/azure.json", "--azure-container-registry-config": "/etc/kubernetes/azure.json", diff --git a/pkg/api/const.go b/pkg/api/const.go index c0a344a8ed..4cccd22e54 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -168,6 +168,8 @@ const ( NVIDIADevicePluginAddonName = "nvidia-device-plugin" // ContainerMonitoringAddonName is the name of the kubernetes Container Monitoring addon deployment ContainerMonitoringAddonName = "container-monitoring" + // IPMASQAgentAddonName is the name of the ip masq agent addon + IPMASQAgentAddonName = "ip-masq-agent" // DefaultPrivateClusterEnabled determines the acs-engine provided default for enabling kubernetes Private Cluster DefaultPrivateClusterEnabled = false // NetworkPolicyAzure is the string expression for Azure CNI network policy manager diff --git a/pkg/api/types.go b/pkg/api/types.go index e550682082..60ecf26b0b 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1204,6 +1204,11 @@ func (k *KubernetesConfig) IsDashboardEnabled() bool { return k.isAddonEnabled(DefaultDashboardAddonName, DefaultDashboardAddonEnabled) } +// IsIPMasqAgentEnabled checks if the ip-masq-agent addon is enabled +func (k *KubernetesConfig) IsIPMasqAgentEnabled() bool { + return k.isAddonEnabled(IPMASQAgentAddonName, true) +} + // IsNSeriesSKU returns whether or not the agent pool has Standard_N SKU VMs func IsNSeriesSKU(p *Properties) bool { for _, profile := range p.AgentPoolProfiles { diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index 9e418b5965..0ce2f7d600 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -1374,6 +1374,37 @@ func TestIsMetricsServerEnabled(t *testing.T) { } } +func TestIsIPMasqAgentEnabled(t *testing.T) { + c := KubernetesConfig{ + Addons: []KubernetesAddon{ + getMockAddon("addon"), + }, + } + enabled := c.IsIPMasqAgentEnabled() + enabledDefault := true + if enabled != enabledDefault { + t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return %t when no ip-masq-agent addon has been specified, instead returned %t", enabledDefault, enabled) + } + c.Addons = append(c.Addons, getMockAddon(IPMASQAgentAddonName)) + enabled = c.IsIPMasqAgentEnabled() + if !enabled { + t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return true when ip-masq-agent adddon has been specified, instead returned %t", enabled) + } + b := false + c = KubernetesConfig{ + Addons: []KubernetesAddon{ + { + Name: IPMASQAgentAddonName, + Enabled: &b, + }, + }, + } + enabled = c.IsIPMasqAgentEnabled() + if enabled { + t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return false when ip-masq-agent addon has been specified as disabled, instead returned %t", enabled) + } +} + func TestCloudProviderDefaults(t *testing.T) { // Test cloudprovider defaults when no user-provided values v := "1.8.0" From bb18c907519bd0dde23e1e8088512dfbe46996d2 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 27 Sep 2018 15:21:21 -0700 Subject: [PATCH 02/15] vmss fix, unit test --- parts/k8s/kubernetesmastercustomdatavmss.yml | 6 ++++++ pkg/api/vlabs/validate.go | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml index b43b7ed392..971eca8b38 100644 --- a/parts/k8s/kubernetesmastercustomdatavmss.yml +++ b/parts/k8s/kubernetesmastercustomdatavmss.yml @@ -394,6 +394,12 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" {{end}} + sed -i "s||{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + - path: "/opt/azure/containers/provision.sh" permissions: "0744" encoding: gzip diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 4dea9bb398..07b8a8acd6 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -1075,11 +1075,12 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows bool) error { } } } - if _, ok := k.KubeletConfig["--non-masquerade-cidr"]; ok { + // Re-enable this unit test if --non-masquerade-cidr is re-introduced + /*if _, ok := k.KubeletConfig["--non-masquerade-cidr"]; ok { if _, _, err := net.ParseCIDR(k.KubeletConfig["--non-masquerade-cidr"]); err != nil { return errors.Errorf("--non-masquerade-cidr kubelet config '%s' is an invalid CIDR string", k.KubeletConfig["--non-masquerade-cidr"]) } - } + }*/ } if _, ok := k.ControllerManagerConfig["--pod-eviction-timeout"]; ok { From f175b62f6df8cb14ca92d581c9c216fd73307d0f Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 27 Sep 2018 15:33:14 -0700 Subject: [PATCH 03/15] tests --- pkg/api/vlabs/validate_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 41a042816a..468e45678b 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -414,14 +414,15 @@ func Test_KubernetesConfig_Validate(t *testing.T) { t.Error("should not error on valid --non-masquerade-cidr") } - c = KubernetesConfig{ + // Re-implement these tests if we re-introduce --ip-maquerade-cidr + /*c = KubernetesConfig{ KubeletConfig: map[string]string{ "--non-masquerade-cidr": "10.120.1.0/invalid", }, } if err := c.Validate(k8sVersion, false); err == nil { t.Error("should error on invalid --non-masquerade-cidr") - } + }*/ c = KubernetesConfig{ MaxPods: KubernetesMinMaxPods - 1, From 75051c0f4408aa71e1596f7385b32d88da789210 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 27 Sep 2018 15:50:47 -0700 Subject: [PATCH 04/15] add 168.63.129.16/32 to nonMasqueradeCIDRs list --- parts/k8s/addons/ip-masq-agent.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/k8s/addons/ip-masq-agent.yaml b/parts/k8s/addons/ip-masq-agent.yaml index fb2a7b424f..65f339a9ed 100644 --- a/parts/k8s/addons/ip-masq-agent.yaml +++ b/parts/k8s/addons/ip-masq-agent.yaml @@ -50,6 +50,6 @@ metadata: data: ip-masq-agent: |- nonMasqueradeCIDRs: - + ,168.63.129.16/32 masqLinkLocal: true resyncInterval: 60s \ No newline at end of file From 948fb5538670f1f66ce4bda167423420da8235c3 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 27 Sep 2018 15:53:58 -0700 Subject: [PATCH 05/15] reformat --- parts/k8s/addons/ip-masq-agent.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parts/k8s/addons/ip-masq-agent.yaml b/parts/k8s/addons/ip-masq-agent.yaml index 65f339a9ed..e0b4715662 100644 --- a/parts/k8s/addons/ip-masq-agent.yaml +++ b/parts/k8s/addons/ip-masq-agent.yaml @@ -50,6 +50,7 @@ metadata: data: ip-masq-agent: |- nonMasqueradeCIDRs: - ,168.63.129.16/32 + - + - 168.63.129.16/32 masqLinkLocal: true resyncInterval: 60s \ No newline at end of file From fcceeb4cf34dac2b2a19a6f2a2911eb8547f346f Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 27 Sep 2018 16:58:43 -0700 Subject: [PATCH 06/15] containers need names! --- parts/k8s/kubernetesparams.t | 2 ++ pkg/acsengine/addons.go | 1 + 2 files changed, 3 insertions(+) diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 4bfcdebe4c..d71718b975 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -577,6 +577,7 @@ "type": "string" }, {{end}} +{{if .OrchestratorProfile.KubernetesConfig.IsIPMasqAgentEnabled}} "kubernetesIPMasqAgentCPURequests": { "metadata": { "description": "IP Masq Agent CPU Requests" @@ -601,6 +602,7 @@ }, "type": "string" }, +{{end}} "kubernetesPodInfraContainerSpec": { "metadata": { "description": "The container spec for pod infra." diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index 40b6d2c24f..f99d21032e 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -184,6 +184,7 @@ func setAddonsConfig(cs *api.ContainerService) { Enabled: helpers.PointerToBool(true), Containers: []api.KubernetesContainerSpec{ { + Name: IPMASQAgentAddonName, CPURequests: "50m", MemoryRequests: "10Mi", CPULimits: "50m", From 7084e5d021a7dd8d6616856363b0973263815650 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 28 Sep 2018 09:34:41 -0700 Subject: [PATCH 07/15] use VNET CIDR instead of subnet CIDR --- parts/k8s/kubernetesparams.t | 2 +- pkg/acsengine/const.go | 2 ++ pkg/acsengine/template_generator.go | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index d71718b975..50db043ac0 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -721,7 +721,7 @@ "type": "int" }, "vnetCidr": { - "defaultValue": "10.0.0.0/8", + "defaultValue": {{GetDefaultVNETCIDR}}, "metadata": { "description": "Cluster vnet cidr" }, diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 972f00ad4a..a5c7addacd 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -31,6 +31,8 @@ const ( // DefaultKubernetesSubnet specifies the default subnet used for all masters, agents and pods // when VNET integration is enabled. DefaultKubernetesSubnet = "10.240.0.0/12" + // DefaultVNETCIDR is the default CIDR block for the VNET + DefaultVNETCIDR = "10.0.0.0/8" // DefaultKubernetesMaxPods is the maximum number of pods to run on a node. DefaultKubernetesMaxPods = 110 // DefaultKubernetesMaxPodsVNETIntegrated is the maximum number of pods to run on a node when VNET integration is enabled. diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index d5a3581d5e..07fbb4ab93 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -522,6 +522,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } return GetMasterAgentAllowedSizes() }, + "GetDefaultVNETCIDR": func() string { + return DefaultVNETCIDR + }, "GetAgentAllowedSizes": func() string { if cs.Properties.OrchestratorProfile.IsKubernetes() || cs.Properties.OrchestratorProfile.IsOpenShift() { return GetKubernetesAgentAllowedSizes() From 39f7a25b4251c3e1455f01bdb905ac4e237c1c13 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 28 Sep 2018 09:50:12 -0700 Subject: [PATCH 08/15] wrap in double-quotes and default val --- parts/k8s/kubernetesparams.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 50db043ac0..3369baa02f 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -205,6 +205,7 @@ "metadata": { "description": "kubernetesNonMasqueradeCidr cluster subnet" }, + "defaultValue": "{{GetDefaultVNETCIDR}}", "type": "string" }, "kubernetesKubeletClusterDomain": { @@ -721,7 +722,7 @@ "type": "int" }, "vnetCidr": { - "defaultValue": {{GetDefaultVNETCIDR}}, + "defaultValue": "{{GetDefaultVNETCIDR}}", "metadata": { "description": "Cluster vnet cidr" }, From b638e643e98a6784e5234c245a9bcfb8b1408ab0 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 28 Sep 2018 14:38:28 -0700 Subject: [PATCH 09/15] distinct azure cni / kubenet implementation --- parts/k8s/addons/ip-masq-agent.yaml | 4 ++-- parts/k8s/kubernetesmastercustomdata.yml | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/parts/k8s/addons/ip-masq-agent.yaml b/parts/k8s/addons/ip-masq-agent.yaml index e0b4715662..83cd71f27f 100644 --- a/parts/k8s/addons/ip-masq-agent.yaml +++ b/parts/k8s/addons/ip-masq-agent.yaml @@ -51,6 +51,6 @@ data: ip-masq-agent: |- nonMasqueradeCIDRs: - - - 168.63.129.16/32 - masqLinkLocal: true + - + masqLinkLocal: resyncInterval: 60s \ No newline at end of file diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index f6de0e9d0c..87e44f78db 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -397,6 +397,13 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" +{{if IsAzureCNI}} + sed -i "s||168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" +{{else}} + sed -i "s||d" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" +{{end}} - path: "/opt/azure/containers/provision.sh" permissions: "0744" From c06bd16740890873f9f0e9301e570ec8e49f7c0b Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 28 Sep 2018 15:47:45 -0700 Subject: [PATCH 10/15] correct sed --- parts/k8s/kubernetesmastercustomdata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 87e44f78db..19605fd96d 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -401,7 +401,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" sed -i "s||true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" {{else}} - sed -i "s||d" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "\||d" "/etc/kubernetes/addons/ip-masq-agent.yaml" sed -i "s||false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" {{end}} From f6be7dbf163d9965c4d0eae59df813ecf6f29851 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 28 Sep 2018 16:48:15 -0700 Subject: [PATCH 11/15] add e2e test --- test/e2e/kubernetes/kubernetes_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index 0e37106521..33a62e5a81 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -427,6 +427,25 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } }) + It("should have ip-masq-agent running", func() { + if hasIPMasqAgent, IPMasqAgentAddon := eng.HasAddon("ip-masq-agent"); hasIPMasqAgent { + running, err := pod.WaitOnReady("azure-ip-masq-agent", "kube-system", 3, 30*time.Second, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(running).To(Equal(true)) + By("Ensuring that the correct resources have been applied") + pods, err := pod.GetAllByPrefix("azure-ip-masq-agent", "kube-system") + Expect(err).NotTo(HaveOccurred()) + for _, p := range pods { + for i, c := range IPMasqAgentAddon.Containers { + err := p.Spec.Containers[i].ValidateResources(c) + Expect(err).NotTo(HaveOccurred()) + } + } + } else { + Skip("ip-masq-agent disabled for this cluster, will not test") + } + }) + It("should have aci-connector running", func() { if hasACIConnector, ACIConnectorAddon := eng.HasAddon("aci-connector"); hasACIConnector { running, err := pod.WaitOnReady("aci-connector", "kube-system", 3, 30*time.Second, cfg.Timeout) From 0de129203ef0dfc9812a146c2748cfd542213151 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 28 Sep 2018 17:15:44 -0700 Subject: [PATCH 12/15] requests/limits --- pkg/acsengine/addons.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index f99d21032e..b120bac277 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -186,9 +186,9 @@ func setAddonsConfig(cs *api.ContainerService) { { Name: IPMASQAgentAddonName, CPURequests: "50m", - MemoryRequests: "10Mi", + MemoryRequests: "50Mi", CPULimits: "50m", - MemoryLimits: "10Mi", + MemoryLimits: "250Mi", }, }, } From 456b944573b2213fb1b77f2c5610aa9363faac69 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 3 Oct 2018 11:14:49 -0600 Subject: [PATCH 13/15] fix omissions in vmss master --- parts/k8s/kubernetesmastercustomdatavmss.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml index 971eca8b38..caa0f7317b 100644 --- a/parts/k8s/kubernetesmastercustomdatavmss.yml +++ b/parts/k8s/kubernetesmastercustomdatavmss.yml @@ -399,6 +399,13 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" +{{if IsAzureCNI}} + sed -i "s||168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" +{{else}} + sed -i "\||d" "/etc/kubernetes/addons/ip-masq-agent.yaml" + sed -i "s||false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" +{{end}} - path: "/opt/azure/containers/provision.sh" permissions: "0744" From 921a59578f3703f72b2e75873bf2a1531936027c Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 3 Oct 2018 11:31:38 -0600 Subject: [PATCH 14/15] enable ip-masq-agent params --- pkg/acsengine/params_k8s.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 7b033ebe3a..95975039d0 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -219,6 +219,16 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params } } } + if kubernetesConfig.IsIPMasqAgentEnabled() { + ipMasqAgentAddon := kubernetesConfig.GetAddonByName(IPMASQAgentAddonName) + i := ipMasqAgentAddon.GetAddonContainersIndexByName(IPMASQAgentAddonName) + if i > -1 { + addValue(parametersMap, "kubernetesIPMasqAgentCPURequests", ipMasqAgentAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesIPMasqAgentMemoryRequests", ipMasqAgentAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesIPMasqAgentCPULimit", ipMasqAgentAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesIPMasqAgentMemoryLimit", ipMasqAgentAddon.Containers[c].MemoryLimits) + } + } if kubernetesConfig.LoadBalancerSku == "Standard" { random := rand.New(rand.NewSource(time.Now().UnixNano())) elbsvcName := random.Int() @@ -249,7 +259,15 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params CloudProviderRateLimitBucket: kubernetesConfig.CloudProviderRateLimitBucket, }) addValue(parametersMap, "kubeClusterCidr", kubernetesConfig.ClusterSubnet) - addValue(parametersMap, "kubernetesNonMasqueradeCidr", kubernetesConfig.KubeletConfig["--non-masquerade-cidr"]) + if properties.OrchestratorProfile.IsAzureCNI() { + if properties.MasterProfile != nil && properties.MasterProfile.IsCustomVNET() { + addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.MasterProfile.VnetCidr) + } else { + addValue(parametersMap, "kubernetesNonMasqueradeCidr", DefaultVNETCIDR) + } + } else { + addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet) + } addValue(parametersMap, "kubernetesKubeletClusterDomain", kubernetesConfig.KubeletConfig["--cluster-domain"]) addValue(parametersMap, "dockerBridgeCidr", kubernetesConfig.DockerBridgeSubnet) addValue(parametersMap, "networkPolicy", kubernetesConfig.NetworkPolicy) From c2371943b24c697d20ed45b91a7bb112775f0607 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 3 Oct 2018 11:36:41 -0600 Subject: [PATCH 15/15] enable addon enforcement as const --- pkg/acsengine/addons.go | 2 +- pkg/api/const.go | 2 ++ pkg/api/types.go | 2 +- pkg/api/types_test.go | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index b120bac277..7f96be98f1 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -181,7 +181,7 @@ func setAddonsConfig(cs *api.ContainerService) { defaultIPMasqAgentAddonsConfig := api.KubernetesAddon{ Name: IPMASQAgentAddonName, - Enabled: helpers.PointerToBool(true), + Enabled: helpers.PointerToBool(api.IPMasqAgentAddonEnabled), Containers: []api.KubernetesContainerSpec{ { Name: IPMASQAgentAddonName, diff --git a/pkg/api/const.go b/pkg/api/const.go index 4cccd22e54..7faec588a6 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -144,6 +144,8 @@ const ( DefaultContainerMonitoringAddonEnabled = false // DefaultAzureCNINetworkMonitoringAddonEnabled Azure CNI networkmonitor addon default DefaultAzureCNINetworkMonitoringAddonEnabled = false + // IPMasqAgentAddonEnabled enables the ip-masq-agent addon + IPMasqAgentAddonEnabled = true // DefaultTillerAddonName is the name of the tiller addon deployment DefaultTillerAddonName = "tiller" // DefaultAADPodIdentityAddonName is the name of the aad-pod-identity addon deployment diff --git a/pkg/api/types.go b/pkg/api/types.go index 60ecf26b0b..4f969f5cb4 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1206,7 +1206,7 @@ func (k *KubernetesConfig) IsDashboardEnabled() bool { // IsIPMasqAgentEnabled checks if the ip-masq-agent addon is enabled func (k *KubernetesConfig) IsIPMasqAgentEnabled() bool { - return k.isAddonEnabled(IPMASQAgentAddonName, true) + return k.isAddonEnabled(IPMASQAgentAddonName, IPMasqAgentAddonEnabled) } // IsNSeriesSKU returns whether or not the agent pool has Standard_N SKU VMs diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index 0ce2f7d600..4bce50f8e9 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -1381,7 +1381,7 @@ func TestIsIPMasqAgentEnabled(t *testing.T) { }, } enabled := c.IsIPMasqAgentEnabled() - enabledDefault := true + enabledDefault := IPMasqAgentAddonEnabled if enabled != enabledDefault { t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return %t when no ip-masq-agent addon has been specified, instead returned %t", enabledDefault, enabled) }