diff --git a/main.go b/main.go index 1a47afe4a4..cb53df67a7 100644 --- a/main.go +++ b/main.go @@ -83,6 +83,7 @@ type CmdLineOpts struct { version bool kubeSubnetMgr bool kubeApiUrl string + kubeAnnotationPrefix string kubeConfigFile string iface flagSlice ifaceRegex flagSlice @@ -121,6 +122,7 @@ func init() { flannelFlags.BoolVar(&opts.ipMasq, "ip-masq", false, "setup IP masquerade rule for traffic destined outside of overlay network") flannelFlags.BoolVar(&opts.kubeSubnetMgr, "kube-subnet-mgr", false, "contact the Kubernetes API for subnet assignment instead of etcd.") flannelFlags.StringVar(&opts.kubeApiUrl, "kube-api-url", "", "Kubernetes API server URL. Does not need to be specified if flannel is running in a pod.") + flannelFlags.StringVar(&opts.kubeAnnotationPrefix, "kube-annotation-prefix", "flannel.alpha.coreos.com", `Kubernetes annotation prefix. Can contain single slash "/", otherwise it will be appended at the end.`) flannelFlags.StringVar(&opts.kubeConfigFile, "kubeconfig-file", "", "kubeconfig file location. Does not need to be specified if flannel is running in a pod.") flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit") flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen") @@ -155,7 +157,7 @@ func usage() { func newSubnetManager() (subnet.Manager, error) { if opts.kubeSubnetMgr { - return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile) + return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile, opts.kubeAnnotationPrefix) } cfg := &etcdv2.EtcdConfig{ diff --git a/subnet/kube/annotations.go b/subnet/kube/annotations.go new file mode 100644 index 0000000000..e76b8e532f --- /dev/null +++ b/subnet/kube/annotations.go @@ -0,0 +1,66 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kube + +import ( + "errors" + "regexp" + "strings" +) + +type annotations struct { + SubnetKubeManaged string + BackendData string + BackendType string + BackendPublicIP string + BackendPublicIPOverwrite string +} + +func newAnnotations(prefix string) (annotations, error) { + slashCnt := strings.Count(prefix, "/") + if slashCnt > 1 { + return annotations{}, errors.New("subnet/kube: prefix can contain at most single slash") + } + if slashCnt == 0 { + prefix += "/" + } + if !strings.HasSuffix(prefix, "/") && !strings.HasSuffix(prefix, "-") { + prefix += "-" + } + + // matches is a regexp matching the format used by the kubernetes for + // annotations. Following rules apply: + // + // - must start with FQDN - must contain at most one slash "/" + // - must contain only lowercase letters, nubers, underscores, + // hyphens, dots and slash + matches, err := regexp.MatchString(`(?:[a-z0-9_-]+\.)+[a-z0-9_-]+/(?:[a-z0-9_-]+-)?$`, prefix) + if err != nil { + panic(err) + } + if !matches { + return annotations{}, errors.New("subnet/kube: prefix must be in a format: fqdn/[0-9a-z-_]*") + } + + a := annotations{ + SubnetKubeManaged: prefix + "kube-subnet-manager", + BackendData: prefix + "backend-data", + BackendType: prefix + "backend-type", + BackendPublicIP: prefix + "public-ip", + BackendPublicIPOverwrite: prefix + "public-ip-overwrite", + } + + return a, nil +} diff --git a/subnet/kube/annotations_test.go b/subnet/kube/annotations_test.go new file mode 100644 index 0000000000..ed889a4ced --- /dev/null +++ b/subnet/kube/annotations_test.go @@ -0,0 +1,96 @@ +// Copyright 2018 flannel authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kube + +import "testing" + +func Test_newAnnotations(t *testing.T) { + testCases := []struct { + prefix string + expectedBackendType string + hasError bool + }{ + { + prefix: "flannel.alpha.coreos.com", + expectedBackendType: "flannel.alpha.coreos.com/backend-type", + }, + { + prefix: "flannel.alpha.coreos.com/", + expectedBackendType: "flannel.alpha.coreos.com/backend-type", + }, + { + prefix: "flannel.alpha.coreos.com/prefix", + expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type", + }, + { + prefix: "flannel.alpha.coreos.com/prefix-", + expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type", + }, + { + prefix: "org.com", + expectedBackendType: "org.com/backend-type", + }, + { + prefix: "org9.com", + expectedBackendType: "org9.com/backend-type", + }, + { + prefix: "org.com/9", + expectedBackendType: "org.com/9-backend-type", + }, + { + // Not a fqdn. + prefix: "org", + hasError: true, + }, + { + // Not a fqdn before /. + prefix: "org/", + hasError: true, + }, + { + // Not a fqdn before /. + prefix: "org/prefix", + hasError: true, + }, + { + // Uppercase letters. + prefix: "org.COM", + hasError: true, + }, + { + // Uppercase letters. + prefix: "org.com/PREFIX", + hasError: true, + }, + } + + for i, tc := range testCases { + as, err := newAnnotations(tc.prefix) + + if err != nil && !tc.hasError { + t.Errorf("#%d: error = %s, want nil", i, err) + continue + } + if err == nil && tc.hasError { + t.Errorf("#%d: error = nil, want non-nil", i) + continue + } + + if as.BackendType != tc.expectedBackendType { + t.Errorf("#%d: BackendType = %s, want %s", i, as.BackendType, tc.expectedBackendType) + } + } +} diff --git a/subnet/kube/kube.go b/subnet/kube/kube.go index 38375d8d03..502c488ecb 100644 --- a/subnet/kube/kube.go +++ b/subnet/kube/kube.go @@ -51,16 +51,11 @@ const ( resyncPeriod = 5 * time.Minute nodeControllerSyncTimeout = 10 * time.Minute - subnetKubeManagedAnnotation = "flannel.alpha.coreos.com/kube-subnet-manager" - backendDataAnnotation = "flannel.alpha.coreos.com/backend-data" - backendTypeAnnotation = "flannel.alpha.coreos.com/backend-type" - backendPublicIPAnnotation = "flannel.alpha.coreos.com/public-ip" - backendPublicIPOverwriteAnnotation = "flannel.alpha.coreos.com/public-ip-overwrite" - netConfPath = "/etc/kube-flannel/net-conf.json" ) type kubeSubnetManager struct { + annotations annotations client clientset.Interface nodeName string nodeStore listers.NodeLister @@ -69,7 +64,7 @@ type kubeSubnetManager struct { events chan subnet.Event } -func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) { +func NewSubnetManager(apiUrl, kubeconfig, prefix string) (subnet.Manager, error) { var cfg *rest.Config var err error @@ -122,7 +117,7 @@ func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) { return nil, fmt.Errorf("error parsing subnet config: %s", err) } - sm, err := newKubeSubnetManager(c, sc, nodeName) + sm, err := newKubeSubnetManager(c, sc, nodeName, prefix) if err != nil { return nil, fmt.Errorf("error creating network manager: %s", err) } @@ -140,8 +135,13 @@ func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) { return sm, nil } -func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName string) (*kubeSubnetManager, error) { +func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName, prefix string) (*kubeSubnetManager, error) { + var err error var ksm kubeSubnetManager + ksm.annotations, err = newAnnotations(prefix) + if err != nil { + return nil, err + } ksm.client = c ksm.nodeName = nodeName ksm.subnetConf = sc @@ -175,11 +175,11 @@ func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName str func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj interface{}) { n := obj.(*v1.Node) - if s, ok := n.Annotations[subnetKubeManagedAnnotation]; !ok || s != "true" { + if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" { return } - l, err := nodeToLease(*n) + l, err := ksm.nodeToLease(*n) if err != nil { glog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err) return @@ -190,16 +190,16 @@ func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj inter func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{}) { o := oldObj.(*v1.Node) n := newObj.(*v1.Node) - if s, ok := n.Annotations[subnetKubeManagedAnnotation]; !ok || s != "true" { + if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" { return } - if o.Annotations[backendDataAnnotation] == n.Annotations[backendDataAnnotation] && - o.Annotations[backendTypeAnnotation] == n.Annotations[backendTypeAnnotation] && - o.Annotations[backendPublicIPAnnotation] == n.Annotations[backendPublicIPAnnotation] { + if o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] && + o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] && + o.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] { return // No change to lease } - l, err := nodeToLease(*n) + l, err := ksm.nodeToLease(*n) if err != nil { glog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err) return @@ -233,24 +233,24 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le if err != nil { return nil, err } - if n.Annotations[backendDataAnnotation] != string(bd) || - n.Annotations[backendTypeAnnotation] != attrs.BackendType || - n.Annotations[backendPublicIPAnnotation] != attrs.PublicIP.String() || - n.Annotations[subnetKubeManagedAnnotation] != "true" || - (n.Annotations[backendPublicIPOverwriteAnnotation] != "" && n.Annotations[backendPublicIPOverwriteAnnotation] != attrs.PublicIP.String()) { - n.Annotations[backendTypeAnnotation] = attrs.BackendType - n.Annotations[backendDataAnnotation] = string(bd) - if n.Annotations[backendPublicIPOverwriteAnnotation] != "" { - if n.Annotations[backendPublicIPAnnotation] != n.Annotations[backendPublicIPOverwriteAnnotation] { + if n.Annotations[ksm.annotations.BackendData] != string(bd) || + n.Annotations[ksm.annotations.BackendType] != attrs.BackendType || + n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() || + n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" || + (n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String()) { + n.Annotations[ksm.annotations.BackendType] = attrs.BackendType + n.Annotations[ksm.annotations.BackendData] = string(bd) + if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" { + if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] { glog.Infof("Overriding public ip with '%s' from node annotation '%s'", - n.Annotations[backendPublicIPOverwriteAnnotation], - backendPublicIPOverwriteAnnotation) - n.Annotations[backendPublicIPAnnotation] = n.Annotations[backendPublicIPOverwriteAnnotation] + n.Annotations[ksm.annotations.BackendPublicIPOverwrite], + ksm.annotations.BackendPublicIPOverwrite) + n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite] } } else { - n.Annotations[backendPublicIPAnnotation] = attrs.PublicIP.String() + n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String() } - n.Annotations[subnetKubeManagedAnnotation] = "true" + n.Annotations[ksm.annotations.SubnetKubeManaged] = "true" oldData, err := json.Marshal(cachedNode) if err != nil { @@ -295,14 +295,14 @@ func (ksm *kubeSubnetManager) Run(ctx context.Context) { ksm.nodeController.Run(ctx.Done()) } -func nodeToLease(n v1.Node) (l subnet.Lease, err error) { - l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[backendPublicIPAnnotation]) +func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error) { + l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP]) if err != nil { return l, err } - l.Attrs.BackendType = n.Annotations[backendTypeAnnotation] - l.Attrs.BackendData = json.RawMessage(n.Annotations[backendDataAnnotation]) + l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType] + l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData]) _, cidr, err := net.ParseCIDR(n.Spec.PodCIDR) if err != nil {