From 5c7091b4c95008f5f2b516d0d98a83b0f5580680 Mon Sep 17 00:00:00 2001 From: ksubrmnn Date: Tue, 21 May 2019 16:19:28 -0700 Subject: [PATCH] EnableNonPersistent flag for Windows Overlay networks --- backend/vxlan/device_windows.go | 217 ++++++++++++-------------------- 1 file changed, 83 insertions(+), 134 deletions(-) diff --git a/backend/vxlan/device_windows.go b/backend/vxlan/device_windows.go index fc8d1a9744..f350521442 100644 --- a/backend/vxlan/device_windows.go +++ b/backend/vxlan/device_windows.go @@ -16,10 +16,7 @@ package vxlan import ( "encoding/json" - "fmt" - "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" - "github.com/buger/jsonparser" "github.com/coreos/flannel/pkg/ip" log "github.com/golang/glog" "github.com/juju/errors" @@ -37,19 +34,52 @@ type vxlanDeviceAttrs struct { } type vxlanDevice struct { - link *hcsshim.HNSNetwork + link *hcn.HostComputeNetwork macPrefix string directRouting bool } func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) { - hnsNetwork := &hcsshim.HNSNetwork{ - Name: devAttrs.name, - Type: "Overlay", - Subnets: make([]hcsshim.Subnet, 0, 1), + subnet := createSubnet(devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String(), "0.0.0.0/0") + network := &hcn.HostComputeNetwork{ + Type: "Overlay", + Name: devAttrs.name, + Ipams: []hcn.Ipam{ + { + Type: "Static", + Subnets: []hcn.Subnet{ + *subnet, + }, + }, + }, + Flags: hcn.EnableNonPersistent, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + } + + vsid := &hcn.VsidPolicySetting{ + IsolationId: devAttrs.vni, + } + vsidJson, err := json.Marshal(vsid) + if err != nil { + return nil, err + } + + sp := &hcn.SubnetPolicy{ + Type: hcn.VSID, } + sp.Settings = vsidJson - hnsNetwork, err := ensureNetwork(hnsNetwork, int64(devAttrs.vni), devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String()) + spJson, err := json.Marshal(sp) + if err != nil { + return nil, err + } + + network.Ipams[0].Subnets[0].Policies = append(network.Ipams[0].Subnets[0].Policies, spJson) + + hnsNetwork, err := ensureNetwork(network, devAttrs.addressPrefix.String()) if err != nil { return nil, err } @@ -59,20 +89,17 @@ func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) { }, nil } -func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expectedAddressPrefix, expectedGW string) (*hcsshim.HNSNetwork, error) { +func ensureNetwork(expectedNetwork *hcn.HostComputeNetwork, expectedAddressPrefix string) (*hcn.HostComputeNetwork, error) { createNetwork := true networkName := expectedNetwork.Name - // 1. Check if the HNSNetwork exists and has the expected settings - existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + // 1. Check if the HostComputeNetwork exists and has the expected settings + existingNetwork, err := hcn.GetNetworkByName(networkName) if err == nil { if existingNetwork.Type == expectedNetwork.Type { - for _, existingSubnet := range existingNetwork.Subnets { - if existingSubnet.AddressPrefix == expectedAddressPrefix && existingSubnet.GatewayAddress == expectedGW { - createNetwork = false - log.Infof("Found existing HNSNetwork %s", networkName) - break - } + if existingNetwork.Ipams[0].Subnets[0].IpAddressPrefix == expectedAddressPrefix { + createNetwork = false + log.Infof("Found existing HostComputeNetwork %s", networkName) } } } @@ -80,66 +107,47 @@ func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expe // 2. Create a new HNSNetwork if createNetwork { if existingNetwork != nil { - if _, err := existingNetwork.Delete(); err != nil { - return nil, errors.Annotatef(err, "failed to delete existing HNSNetwork %s", networkName) + if err := existingNetwork.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete existing HostComputeNetwork %s", networkName) } - log.Infof("Deleted stale HNSNetwork %s", networkName) + log.Infof("Deleted stale HostComputeNetwork %s", networkName) } - // Add a VxLan subnet - expectedNetwork.Subnets = append(expectedNetwork.Subnets, hcsshim.Subnet{ - AddressPrefix: expectedAddressPrefix, - GatewayAddress: expectedGW, - Policies: []json.RawMessage{ - []byte(fmt.Sprintf(`{"Type":"VSID","VSID":%d}`, expectedVSID)), - }, - }) - - // Config request params - jsonRequest, err := json.Marshal(expectedNetwork) + log.Infof("Attempting to create HostComputeNetwork %v", expectedNetwork) + newNetwork, err := expectedNetwork.Create() if err != nil { - return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork) - } - - log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest)) - newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest)) - if err != nil { - return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName) + return nil, errors.Annotatef(err, "failed to create HostComputeNetwork %s", networkName) } var waitErr, lastErr error // Wait for the network to populate Management IP - log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName) + log.Infof("Waiting to get ManagementIP from HostComputeNetwork %s", networkName) waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) { - newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "") - return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil + newNetwork, lastErr = hcn.GetNetworkByID(newNetwork.Id) + return newNetwork != nil && len(getManagementIP(newNetwork)) != 0, nil }) if waitErr == wait.ErrWaitTimeout { - return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HNSNetwork %s", networkName) + return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HostComputeNetwork %s", networkName) } + managementIP := getManagementIP(newNetwork) // Wait for the interface with the management IP netshHelper := netsh.New(utilexec.New()) - log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP) + log.Infof("Waiting to get net interface for HostComputeNetwork %s (%s)", networkName, managementIP) waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) { - _, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP) + _, lastErr = netshHelper.GetInterfaceByIP(managementIP) return lastErr == nil, nil }) if waitErr == wait.ErrWaitTimeout { - return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP) + return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HostComputeNetwork %s (%s)", networkName, managementIP) } - log.Infof("Created HNSNetwork %s", networkName) + log.Infof("Created HostComputeNetwork %s", networkName) existingNetwork = newNetwork } - existingNetworkV2, err := hcn.GetNetworkByID(existingNetwork.Id) - if err != nil { - return nil, errors.Annotatef(err, "Could not find vxlan0 in V2") - } - addHostRoute := true - for _, policy := range existingNetworkV2.Policies { + for _, policy := range existingNetwork.Policies { if policy.Type == hcn.HostRoute { addHostRoute = false } @@ -153,96 +161,37 @@ func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expe networkRequest := hcn.PolicyNetworkRequest{ Policies: []hcn.NetworkPolicy{hostRoutePolicy}, } - existingNetworkV2.AddPolicy(networkRequest) + err = existingNetwork.AddPolicy(networkRequest) + if err != nil { + log.Infof("Could not apply HostRoute policy for local host to local pod connectivity. This policy requires Windows 18321.1000.19h1_release.190117-1502 or newer") + } } return existingNetwork, nil } -type neighbor struct { - MAC string - IP ip.IP4 - ManagementAddress string -} - -func (dev *vxlanDevice) AddEndpoint(n *neighbor) error { - endpointName := createEndpointName(n.IP) - - // 1. Check if the HNSEndpoint exists and has the expected settings - existingEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName) - if err == nil && existingEndpoint.VirtualNetwork == dev.link.Id { - // Check policies if there is PA type - targetType := "PA" - for _, policy := range existingEndpoint.Policies { - policyType, _ := jsonparser.GetUnsafeString(policy, "Type") - if policyType == targetType { - actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType) - if actualPaIP == n.ManagementAddress { - log.Infof("Found existing remote HNSEndpoint %s", endpointName) - return nil - } +func getManagementIP(network *hcn.HostComputeNetwork) string { + for _, policy := range network.Policies { + if policy.Type == hcn.ProviderAddress { + policySettings := hcn.ProviderAddressEndpointPolicySetting{} + err := json.Unmarshal(policy.Settings, &policySettings) + if err != nil { + return "" } + return policySettings.ProviderAddress } } - - // 2. Create a new HNSNetwork - if existingEndpoint != nil { - if _, err := existingEndpoint.Delete(); err != nil { - return errors.Annotatef(err, "failed to delete existing remote HNSEndpoint %s", endpointName) - } - log.V(4).Infof("Deleted stale HNSEndpoint %s", endpointName) - } - - newEndpoint := &hcsshim.HNSEndpoint{ - Name: endpointName, - IPAddress: n.IP.ToIP(), - MacAddress: n.MAC, - VirtualNetwork: dev.link.Id, - IsRemoteEndpoint: true, - Policies: []json.RawMessage{ - []byte(fmt.Sprintf(`{"Type":"PA","PA":"%s"}`, n.ManagementAddress)), - }, - } - if _, err := newEndpoint.Create(); err != nil { - return errors.Annotatef(err, "failed to create remote HNSEndpoint %s", endpointName) - } - log.V(4).Infof("Created HNSEndpoint %s", endpointName) - - return nil + return "" } -func (dev *vxlanDevice) DelEndpoint(n *neighbor) error { - endpointName := createEndpointName(n.IP) - - existingEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName) - if err == nil && existingEndpoint.VirtualNetwork == dev.link.Id { - // Check policies if there is PA type - targetType := "PA" - for _, policy := range existingEndpoint.Policies { - policyType, _ := jsonparser.GetUnsafeString(policy, "Type") - if policyType == targetType { - actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType) - if actualPaIP == n.ManagementAddress { - // Found it and delete - if _, err := existingEndpoint.Delete(); err != nil { - return errors.Annotatef(err, "failed to delete remote HNSEndpoint %s", endpointName) - } - - log.V(4).Infof("Deleted HNSEndpoint %s", endpointName) - break - } - } - } +func createSubnet(AddressPrefix string, NextHop string, DestPrefix string) *hcn.Subnet { + return &hcn.Subnet{ + IpAddressPrefix: AddressPrefix, + Routes: []hcn.Route{ + { + NextHop: NextHop, + DestinationPrefix: DestPrefix, + }, + }, } - - return nil -} - -func (dev *vxlanDevice) ConjureMac(targetIP ip.IP4) string { - a, b, c, d := targetIP.Octets() - return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", dev.macPrefix, a, b, c, d) -} - -func createEndpointName(targetIP ip.IP4) string { - return "remote_" + targetIP.String() }