ip-masq-agent as addon

parts/k8s/addons/ip-masq-agent.yaml

@@ -0,0 +1,56 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: azure-ip-masq-agent
+  namespace: kube-system
+  labels:
+    component: azure-ip-masq-agent
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    tier: node
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: azure-ip-masq-agent
+        tier: node
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      containers:
+      - name: azure-ip-masq-agent
+        image: gcr.io/google-containers/ip-masq-agent-amd64:v2.0.0
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: azure-ip-masq-agent-config-volume
+            mountPath: /etc/config
+        resources:
+          requests:
+            cpu: <kubernetesIPMasqAgentCPURequests>
+            memory: <kubernetesIPMasqAgentMemoryRequests>
+          limits:
+            cpu: <kubernetesIPMasqAgentCPULimit>
+            memory: <kubernetesIPMasqAgentMemoryLimit>
+      volumes:
+        - name: azure-ip-masq-agent-config-volume
+          configMap:
+            name: azure-ip-masq-agent-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: azure-ip-masq-agent-config
+  namespace: kube-system
+  labels:
+    component: azure-ip-masq-agent
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: EnsureExists
+data:
+  ip-masq-agent: |-
+    nonMasqueradeCIDRs:
+      - <kubernetesNonMasqueradeCidr>
+      - <azureCNINonMasqueradeIP>
+    masqLinkLocal: <masqLinkLocalValue>
+    resyncInterval: 60s

parts/k8s/kubernetesagentcustomdata.yml

@@ -188,10 +188,6 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER
   owner: "root"
   content: |
     #!/bin/bash
-{{if IsAzureCNI}}
-    # SNAT outbound traffic from pods to destinations outside of VNET.
-    iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
-{{end}}
 {{if not EnablePodSecurityPolicy}}
     sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service"
 {{end}}

parts/k8s/kubernetesmastercustomdata.yml

@@ -242,7 +242,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
 
 {{if IsAzureCNI}}
     # SNAT outbound traffic from pods to destinations outside of VNET.
-    iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
     sed -i "s|<azureCNINetworkMonitorImage>|{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml"
 {{end}}
     sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml"
@@ -393,6 +392,19 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     sed -i "s|<kubernetesOMSAgentMemoryLimit>|{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
 {{end}}
 
+    sed -i "s|<kubernetesNonMasqueradeCidr>|{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentCPURequests>|{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentMemoryRequests>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentCPULimit>|{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentMemoryLimit>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+{{if IsAzureCNI}}
+    sed -i "s|<azureCNINonMasqueradeIP>|168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<masqLinkLocalValue>|true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+{{else}}
+    sed -i "\|<azureCNINonMasqueradeIP>|d" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<masqLinkLocalValue>|false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+{{end}}
+
 - path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
   encoding: gzip

parts/k8s/kubernetesmastercustomdatavmss.yml

@@ -244,7 +244,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
 
 {{if IsAzureCNI}}
     # SNAT outbound traffic from pods to destinations outside of VNET.
-    iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
     sed -i "s|<azureCNINetworkMonitorImage>|{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml"
 {{end}}
     sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml"
@@ -395,6 +394,19 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     sed -i "s|<kubernetesOMSAgentMemoryLimit>|{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
 {{end}}
 
+    sed -i "s|<kubernetesNonMasqueradeCidr>|{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentCPURequests>|{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentMemoryRequests>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentCPULimit>|{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<kubernetesIPMasqAgentMemoryLimit>|{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+{{if IsAzureCNI}}
+    sed -i "s|<azureCNINonMasqueradeIP>|168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<masqLinkLocalValue>|true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+{{else}}
+    sed -i "\|<azureCNINonMasqueradeIP>|d" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+    sed -i "s|<masqLinkLocalValue>|false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml"
+{{end}}
+
 - path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
   encoding: gzip

parts/k8s/kubernetesparams.t

@@ -205,6 +205,7 @@
       "metadata": {
         "description": "kubernetesNonMasqueradeCidr cluster subnet"
       },
+      "defaultValue": "{{GetDefaultVNETCIDR}}",
       "type": "string"
     },
     "kubernetesKubeletClusterDomain": {
@@ -576,6 +577,32 @@
       },
       "type": "string"
     },
+{{end}}
+{{if .OrchestratorProfile.KubernetesConfig.IsIPMasqAgentEnabled}}
+    "kubernetesIPMasqAgentCPURequests": {
+      "metadata": {
+        "description": "IP Masq Agent CPU Requests"
+      },
+      "type": "string"
+    },
+    "kubernetesIPMasqAgentMemoryRequests": {
+      "metadata": {
+        "description": "IP Masq Agent Memory Requests"
+      },
+      "type": "string"
+    },
+    "kubernetesIPMasqAgentCPULimit": {
+      "metadata": {
+        "description": "IP Masq Agent CPU Limit"
+      },
+      "type": "string"
+    },
+    "kubernetesIPMasqAgentMemoryLimit": {
+      "metadata": {
+        "description": "IP Masq Agent Memory Limit"
+      },
+      "type": "string"
+    },
 {{end}}
     "kubernetesPodInfraContainerSpec": {
       "metadata": {
@@ -695,7 +722,7 @@
       "type": "int"
     },
     "vnetCidr": {
-      "defaultValue": "10.0.0.0/8",
+      "defaultValue": "{{GetDefaultVNETCIDR}}",
       "metadata": {
         "description": "Cluster vnet cidr"
       },

pkg/acsengine/addons.go

@@ -179,6 +179,20 @@ func setAddonsConfig(cs *api.ContainerService) {
 		},
 	}
 
+	defaultIPMasqAgentAddonsConfig := api.KubernetesAddon{
+		Name:    IPMASQAgentAddonName,
+		Enabled: helpers.PointerToBool(api.IPMasqAgentAddonEnabled),
+		Containers: []api.KubernetesContainerSpec{
+			{
+				Name:           IPMASQAgentAddonName,
+				CPURequests:    "50m",
+				MemoryRequests: "50Mi",
+				CPULimits:      "50m",
+				MemoryLimits:   "250Mi",
+			},
+		},
+	}
+
 	defaultAzureCNINetworkMonitorAddonsConfig := api.KubernetesAddon{
 		Name:    AzureCNINetworkMonitoringAddonName,
 		Enabled: azureCNINetworkMonitorAddonEnabled(o),
@@ -213,6 +227,7 @@ func setAddonsConfig(cs *api.ContainerService) {
 		defaultContainerMonitoringAddonsConfig,
 		defaultAzureCNINetworkMonitorAddonsConfig,
 		defaultAzureNetworkPolicyAddonsConfig,
+		defaultIPMasqAgentAddonsConfig,
 	}
 	// Add default addons specification, if no user-provided spec exists
 	if o.KubernetesConfig.Addons == nil {

pkg/acsengine/artifacts.go

@@ -120,7 +120,6 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti
 			profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultReschedulerAddonName),
 		},
 		{
-
 			kubernetesFeatureSetting{
 				"kubernetesmasteraddons-azure-npm-daemonset.yaml",
 				"azure-npm-daemonset.yaml",
@@ -233,6 +232,14 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti
 			},
 			profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultELBSVCAddonName),
 		},
+		{
+			kubernetesFeatureSetting{
+				"ip-masq-agent.yaml",
+				"ip-masq-agent.yaml",
+				true,
+			},
+			profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(IPMASQAgentAddonName),
+		},
 	}
 }
 

pkg/acsengine/const.go

@@ -31,6 +31,8 @@ const (
 	// DefaultKubernetesSubnet specifies the default subnet used for all masters, agents and pods
 	// when VNET integration is enabled.
 	DefaultKubernetesSubnet = "10.240.0.0/12"
+	// DefaultVNETCIDR is the default CIDR block for the VNET
+	DefaultVNETCIDR = "10.0.0.0/8"
 	// DefaultKubernetesMaxPods is the maximum number of pods to run on a node.
 	DefaultKubernetesMaxPods = 110
 	// DefaultKubernetesMaxPodsVNETIntegrated is the maximum number of pods to run on a node when VNET integration is enabled.
@@ -160,6 +162,8 @@ const (
 	AzureCNINetworkMonitoringAddonName = "azure-cni-networkmonitor"
 	// AzureNetworkPolicyAddonName is the name of the Azure CNI networkmonitor addon
 	AzureNetworkPolicyAddonName = "azure-npm-daemonset"
+	// IPMASQAgentAddonName is the name of the ip masq agent addon
+	IPMASQAgentAddonName = "ip-masq-agent"
 	// DefaultKubernetesKubeletMaxPods is the max pods per kubelet
 	DefaultKubernetesKubeletMaxPods = 110
 	// DefaultMasterEtcdServerPort is the default etcd server port for Kubernetes master nodes

pkg/acsengine/defaults-kubelet.go

@@ -55,7 +55,7 @@ func setKubeletConfig(cs *api.ContainerService) {
 		"--node-status-update-frequency":    KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"],
 		"--image-gc-high-threshold":         strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"--image-gc-low-threshold":          strconv.Itoa(DefaultKubernetesGCLowThreshold),
-		"--non-masquerade-cidr":             o.KubernetesConfig.ClusterSubnet,
+		"--non-masquerade-cidr":             "0.0.0.0",
 		"--cloud-provider":                  "azure",
 		"--cloud-config":                    "/etc/kubernetes/azure.json",
 		"--azure-container-registry-config": "/etc/kubernetes/azure.json",

pkg/acsengine/params_k8s.go

@@ -219,6 +219,16 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 					}
 				}
 			}
+			if kubernetesConfig.IsIPMasqAgentEnabled() {
+				ipMasqAgentAddon := kubernetesConfig.GetAddonByName(IPMASQAgentAddonName)
+				i := ipMasqAgentAddon.GetAddonContainersIndexByName(IPMASQAgentAddonName)
+				if i > -1 {
+					addValue(parametersMap, "kubernetesIPMasqAgentCPURequests", ipMasqAgentAddon.Containers[c].CPURequests)
+					addValue(parametersMap, "kubernetesIPMasqAgentMemoryRequests", ipMasqAgentAddon.Containers[c].MemoryRequests)
+					addValue(parametersMap, "kubernetesIPMasqAgentCPULimit", ipMasqAgentAddon.Containers[c].CPULimits)
+					addValue(parametersMap, "kubernetesIPMasqAgentMemoryLimit", ipMasqAgentAddon.Containers[c].MemoryLimits)
+				}
+			}
 			if kubernetesConfig.LoadBalancerSku == "Standard" {
 				random := rand.New(rand.NewSource(time.Now().UnixNano()))
 				elbsvcName := random.Int()
@@ -249,7 +259,15 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 				CloudProviderRateLimitBucket: kubernetesConfig.CloudProviderRateLimitBucket,
 			})
 			addValue(parametersMap, "kubeClusterCidr", kubernetesConfig.ClusterSubnet)
-			addValue(parametersMap, "kubernetesNonMasqueradeCidr", kubernetesConfig.KubeletConfig["--non-masquerade-cidr"])
+			if properties.OrchestratorProfile.IsAzureCNI() {
+				if properties.MasterProfile != nil && properties.MasterProfile.IsCustomVNET() {
+					addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.MasterProfile.VnetCidr)
+				} else {
+					addValue(parametersMap, "kubernetesNonMasqueradeCidr", DefaultVNETCIDR)
+				}
+			} else {
+				addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet)
+			}
 			addValue(parametersMap, "kubernetesKubeletClusterDomain", kubernetesConfig.KubeletConfig["--cluster-domain"])
 			addValue(parametersMap, "dockerBridgeCidr", kubernetesConfig.DockerBridgeSubnet)
 			addValue(parametersMap, "networkPolicy", kubernetesConfig.NetworkPolicy)

pkg/acsengine/template_generator.go

@@ -522,6 +522,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			}
 			return GetMasterAgentAllowedSizes()
 		},
+		"GetDefaultVNETCIDR": func() string {
+			return DefaultVNETCIDR
+		},
 		"GetAgentAllowedSizes": func() string {
 			if cs.Properties.OrchestratorProfile.IsKubernetes() || cs.Properties.OrchestratorProfile.IsOpenShift() {
 				return GetKubernetesAgentAllowedSizes()

pkg/api/const.go

@@ -144,6 +144,8 @@ const (
 	DefaultContainerMonitoringAddonEnabled = false
 	// DefaultAzureCNINetworkMonitoringAddonEnabled Azure CNI networkmonitor addon default
 	DefaultAzureCNINetworkMonitoringAddonEnabled = false
+	// IPMasqAgentAddonEnabled enables the ip-masq-agent addon
+	IPMasqAgentAddonEnabled = true
 	// DefaultTillerAddonName is the name of the tiller addon deployment
 	DefaultTillerAddonName = "tiller"
 	// DefaultAADPodIdentityAddonName is the name of the aad-pod-identity addon deployment
@@ -168,6 +170,8 @@ const (
 	NVIDIADevicePluginAddonName = "nvidia-device-plugin"
 	// ContainerMonitoringAddonName is the name of the kubernetes Container Monitoring addon deployment
 	ContainerMonitoringAddonName = "container-monitoring"
+	// IPMASQAgentAddonName is the name of the ip masq agent addon
+	IPMASQAgentAddonName = "ip-masq-agent"
 	// DefaultPrivateClusterEnabled determines the acs-engine provided default for enabling kubernetes Private Cluster
 	DefaultPrivateClusterEnabled = false
 	// NetworkPolicyAzure is the string expression for Azure CNI network policy manager

pkg/api/types.go

@@ -1204,6 +1204,11 @@ func (k *KubernetesConfig) IsDashboardEnabled() bool {
 	return k.isAddonEnabled(DefaultDashboardAddonName, DefaultDashboardAddonEnabled)
 }
 
+// IsIPMasqAgentEnabled checks if the ip-masq-agent addon is enabled
+func (k *KubernetesConfig) IsIPMasqAgentEnabled() bool {
+	return k.isAddonEnabled(IPMASQAgentAddonName, IPMasqAgentAddonEnabled)
+}
+
 // IsNSeriesSKU returns whether or not the agent pool has Standard_N SKU VMs
 func IsNSeriesSKU(p *Properties) bool {
 	for _, profile := range p.AgentPoolProfiles {

pkg/api/types_test.go

@@ -1374,6 +1374,37 @@ func TestIsMetricsServerEnabled(t *testing.T) {
 	}
 }
 
+func TestIsIPMasqAgentEnabled(t *testing.T) {
+	c := KubernetesConfig{
+		Addons: []KubernetesAddon{
+			getMockAddon("addon"),
+		},
+	}
+	enabled := c.IsIPMasqAgentEnabled()
+	enabledDefault := IPMasqAgentAddonEnabled
+	if enabled != enabledDefault {
+		t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return %t when no ip-masq-agent addon has been specified, instead returned %t", enabledDefault, enabled)
+	}
+	c.Addons = append(c.Addons, getMockAddon(IPMASQAgentAddonName))
+	enabled = c.IsIPMasqAgentEnabled()
+	if !enabled {
+		t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return true when ip-masq-agent adddon has been specified, instead returned %t", enabled)
+	}
+	b := false
+	c = KubernetesConfig{
+		Addons: []KubernetesAddon{
+			{
+				Name:    IPMASQAgentAddonName,
+				Enabled: &b,
+			},
+		},
+	}
+	enabled = c.IsIPMasqAgentEnabled()
+	if enabled {
+		t.Fatalf("KubernetesConfig.IsIPMasqAgentEnabled() should return false when ip-masq-agent addon has been specified as disabled, instead returned %t", enabled)
+	}
+}
+
 func TestCloudProviderDefaults(t *testing.T) {
 	// Test cloudprovider defaults when no user-provided values
 	v := "1.8.0"

pkg/api/vlabs/validate.go

@@ -1075,11 +1075,12 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows bool) error {
 				}
 			}
 		}
-		if _, ok := k.KubeletConfig["--non-masquerade-cidr"]; ok {
+		// Re-enable this unit test if --non-masquerade-cidr is re-introduced
+		/*if _, ok := k.KubeletConfig["--non-masquerade-cidr"]; ok {
 			if _, _, err := net.ParseCIDR(k.KubeletConfig["--non-masquerade-cidr"]); err != nil {
 				return errors.Errorf("--non-masquerade-cidr kubelet config '%s' is an invalid CIDR string", k.KubeletConfig["--non-masquerade-cidr"])
 			}
-		}
+		}*/
 	}
 
 	if _, ok := k.ControllerManagerConfig["--pod-eviction-timeout"]; ok {

pkg/api/vlabs/validate_test.go

@@ -414,14 +414,15 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
 			t.Error("should not error on valid --non-masquerade-cidr")
 		}
 
-		c = KubernetesConfig{
+		// Re-implement these tests if we re-introduce --ip-maquerade-cidr
+		/*c = KubernetesConfig{
 			KubeletConfig: map[string]string{
 				"--non-masquerade-cidr": "10.120.1.0/invalid",
 			},
 		}
 		if err := c.Validate(k8sVersion, false); err == nil {
 			t.Error("should error on invalid --non-masquerade-cidr")
-		}
+		}*/
 
 		c = KubernetesConfig{
 			MaxPods: KubernetesMinMaxPods - 1,