Add ali cloud VPC network support

Signed-off-by: yaoyao.xyy <yaoyao.xyy@alibaba-inc.com>

Documentation/kube-flannel-aliyun.yml

@@ -0,0 +1,86 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "type": "flannel",
+      "delegate": {
+        "isDefaultGateway": true
+      }
+    }
+  net-conf.json: |
+    {
+      "Network": "10.24.0.0/16",
+      "Backend": {
+        "Type": "ali-vpc"
+      }
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      containers:
+      - name: kube-flannel
+        image: registry.cn-hangzhou.aliyuncs.com/google-containers/flannel-git:v0.6.2-ubuntu
+        command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ACCESS_KEY_ID
+          value: [replace with your own key]
+        - name: ACCESS_KEY_SECRET
+          value: [replaces with your own secret]
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      - name: install-cni
+        image: registry.cn-hangzhou.aliyuncs.com/google-containers/flannel-git:v0.6.2-ubuntu
+        command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ]
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg

README.md

@@ -95,6 +95,15 @@ This is the only mandatory key.
 * alloc: only perform subnet allocation (no forwarding of data packets).
   * `Type` (string): `alloc`
 
+* ali-vpc: create IP routes in a [alicloud VPC route table](https://vpc.console.aliyun.com)
+  * Requirements:
+    * Running on an ECS instance that is in an Alicloud VPC.
+    * Permission require accessid and keysecret 
+  * `Type` (string): `ali-vpc`
+  * `AccessKeyID` (string): api access key id. can also be configure with environment ACCESS_KEY_ID
+  * `AccessKeySecret` (string): api access key secret.can also be configure with environment ACCESS_KEY_SECRET
+  Note: Currently, AliVPC limit the number of entries per route table to 50.  
+
 ### Example configuration JSON
 
 The following configuration illustrates the use of most options with `udp` backend.

backend/alivpc/alivpc.go

@@ -0,0 +1,198 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alivpc
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	log "github.com/golang/glog"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+	"github.com/denverdino/aliyungo/common"
+	"github.com/denverdino/aliyungo/ecs"
+	"github.com/denverdino/aliyungo/metadata"
+)
+
+func init() {
+	backend.Register("ali-vpc", New)
+}
+
+type AliVpcBackend struct {
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
+}
+
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
+	be := AliVpcBackend{
+		sm:       sm,
+		extIface: extIface,
+	}
+	return &be, nil
+}
+
+func (be *AliVpcBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (be *AliVpcBackend) RegisterNetwork(ctx context.Context, network string, config *subnet.Config) (backend.Network, error) {
+	// 1. Parse our configuration
+	cfg := struct {
+		AccessKeyID     string
+		AccessKeySecret string
+	}{}
+
+	if len(config.Backend) > 0 {
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
+			return nil, fmt.Errorf("error decoding VPC backend config: %v", err)
+		}
+	}
+	log.Infof("Unmarshal Configure : %v\n", cfg)
+
+	// 2. Acquire the lease form subnet manager
+	attrs := subnet.LeaseAttrs{
+		PublicIP: ip.FromIP(be.extIface.ExtAddr),
+	}
+
+	l, err := be.sm.AcquireLease(ctx, network, &attrs)
+	switch err {
+	case nil:
+
+	case context.Canceled, context.DeadlineExceeded:
+		return nil, err
+
+	default:
+		return nil, fmt.Errorf("failed to acquire lease: %v", err)
+	}
+	if cfg.AccessKeyID == "" || cfg.AccessKeySecret == "" {
+		cfg.AccessKeyID = os.Getenv("ACCESS_KEY_ID")
+		cfg.AccessKeySecret = os.Getenv("ACCESS_KEY_SECRET")
+
+		if cfg.AccessKeyID == "" || cfg.AccessKeySecret == "" {
+			return nil, fmt.Errorf("ACCESS_KEY_ID and ACCESS_KEY_SECRET must be provided! ")
+		}
+	}
+
+	meta := metadata.NewMetaData(nil)
+	REGION, err := meta.Region()
+	if err != nil {
+		return nil, err
+	}
+	instanceid, err := meta.InstanceID()
+	if err != nil {
+		return nil, err
+	}
+	VpcID, err := meta.VpcID()
+	if err != nil {
+		return nil, err
+	}
+
+	c := ecs.NewClient(cfg.AccessKeyID, cfg.AccessKeySecret)
+
+	vpc, _, err := c.DescribeVpcs(&ecs.DescribeVpcsArgs{
+		RegionId: common.Region(REGION),
+		VpcId:    VpcID,
+	})
+	if err != nil || len(vpc) <= 0 {
+		log.Errorf("Error DescribeVpcs: %s . \n", getErrorString(err))
+		return nil, err
+	}
+
+	vroute, _, err := c.DescribeVRouters(&ecs.DescribeVRoutersArgs{
+		VRouterId: vpc[0].VRouterId,
+		RegionId:  common.Region(REGION),
+	})
+	if err != nil || len(vroute) <= 0 {
+		log.Errorf("Error DescribeVRouters: %s .\n", getErrorString(err))
+		return nil, err
+	}
+	vRouterId := vroute[0].VRouterId
+	rTableId := vroute[0].RouteTableIds.RouteTableId[0]
+
+	rtables, _, err := c.DescribeRouteTables(&ecs.DescribeRouteTablesArgs{
+		VRouterId:    vRouterId,
+		RouteTableId: rTableId,
+	})
+	if err != nil || len(rtables) <= 0 {
+		log.Errorf("Error DescribeRouteTables: %s.\n", err.Error())
+		return nil, err
+	}
+
+	route := &ecs.CreateRouteEntryArgs{
+		DestinationCidrBlock: l.Subnet.String(),
+		NextHopType:          ecs.NextHopIntance,
+		NextHopId:            instanceid,
+		ClientToken:          "",
+		RouteTableId:         rTableId,
+	}
+	if err := be.recreateRoute(c, rtables[0], route); err != nil {
+		return nil, err
+	}
+
+	if err := c.WaitForAllRouteEntriesAvailable(vRouterId, rTableId, 60); err != nil {
+		return nil, err
+	}
+	return &backend.SimpleNetwork{
+		SubnetLease: l,
+		ExtIface:    be.extIface,
+	}, nil
+}
+
+func (be *AliVpcBackend) recreateRoute(c *ecs.Client, table ecs.RouteTableSetType, route *ecs.CreateRouteEntryArgs) error {
+	exist := false
+	for _, e := range table.RouteEntrys.RouteEntry {
+		if e.RouteTableId == route.RouteTableId &&
+			e.Type == ecs.RouteTableCustom &&
+			e.InstanceId == route.NextHopId {
+
+			if e.DestinationCidrBlock == route.DestinationCidrBlock &&
+				e.Status == ecs.RouteEntryStatusAvailable {
+				exist = true
+				log.Infof("Keep target entry: rtableid=%s, CIDR=%s, NextHop=%s \n", e.RouteTableId, e.DestinationCidrBlock, e.InstanceId)
+				continue
+			}
+			// Fix: here we delete all the route which targeted to us(instance) except the specified route.
+			// That means only one CIDR was allowed to target to the instance. Think if We need to change this
+			// to adapt to multi CIDR and deal with unavailable route entry.
+			if err := c.DeleteRouteEntry(&ecs.DeleteRouteEntryArgs{
+				RouteTableId:         route.RouteTableId,
+				DestinationCidrBlock: e.DestinationCidrBlock,
+				NextHopId:            route.NextHopId,
+			}); err != nil {
+				return err
+			}
+
+			log.Infof("Remove old route entry: rtableid=%s, CIDR=%s, NextHop=%s \n", e.RouteTableId, e.DestinationCidrBlock, e.InstanceId)
+			continue
+		}
+
+		log.Infof("Keep route entry: rtableid=%s, CIDR=%s, NextHop=%s \n", e.RouteTableId, e.DestinationCidrBlock, e.InstanceId)
+	}
+	if !exist {
+		return c.CreateRouteEntry(route)
+	}
+	return nil
+}
+
+func getErrorString(e error) string {
+	if e == nil {
+		return ""
+	}
+	return e.Error()
+}

main.go

@@ -40,6 +40,7 @@ import (
 	_ "github.com/coreos/flannel/backend/hostgw"
 	_ "github.com/coreos/flannel/backend/udp"
 	_ "github.com/coreos/flannel/backend/vxlan"
+	_ "github.com/coreos/flannel/backend/alivpc"
 )
 
 type CmdLineOpts struct {