Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow changing subnet/kube annotation prefix #997

Merged
merged 2 commits into from
Oct 5, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ type CmdLineOpts struct {
version bool
kubeSubnetMgr bool
kubeApiUrl string
kubeAnnotationPrefix string
kubeConfigFile string
iface flagSlice
ifaceRegex flagSlice
Expand Down Expand Up @@ -121,6 +122,7 @@ func init() {
flannelFlags.BoolVar(&opts.ipMasq, "ip-masq", false, "setup IP masquerade rule for traffic destined outside of overlay network")
flannelFlags.BoolVar(&opts.kubeSubnetMgr, "kube-subnet-mgr", false, "contact the Kubernetes API for subnet assignment instead of etcd.")
flannelFlags.StringVar(&opts.kubeApiUrl, "kube-api-url", "", "Kubernetes API server URL. Does not need to be specified if flannel is running in a pod.")
flannelFlags.StringVar(&opts.kubeAnnotationPrefix, "kube-annotation-prefix", "flannel.alpha.coreos.com", `Kubernetes annotation prefix. Can contain single slash "/", otherwise it will be appended at the end.`)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reasoning behind supporting both with and without a slash?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My idea was to add a trailing slash if the annotation doesn't contain any. Please check test cases I copied here. They show how the flag value is transformed to the final annotation name.

+		{
+			prefix:              "flannel.alpha.coreos.com",
+			expectedBackendType: "flannel.alpha.coreos.com/backend-type",
+		},
+		{
+			prefix:              "flannel.alpha.coreos.com/",
+			expectedBackendType: "flannel.alpha.coreos.com/backend-type",
+		},
+		{
+			prefix:              "flannel.alpha.coreos.com/prefix",
+			expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type",
+		},
+		{
+			prefix:              "flannel.alpha.coreos.com/prefix-",
+			expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type",
+		},

This would be useful for us. We'd like to to have prefixes like flannel.alpha.coreos.com/some-id-. I'm fine with changing that. Let me know what is your preference.

flannelFlags.StringVar(&opts.kubeConfigFile, "kubeconfig-file", "", "kubeconfig file location. Does not need to be specified if flannel is running in a pod.")
flannelFlags.BoolVar(&opts.version, "version", false, "print version and exit")
flannelFlags.StringVar(&opts.healthzIP, "healthz-ip", "0.0.0.0", "the IP address for healthz server to listen")
Expand Down Expand Up @@ -155,7 +157,7 @@ func usage() {

func newSubnetManager() (subnet.Manager, error) {
if opts.kubeSubnetMgr {
return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile)
return kube.NewSubnetManager(opts.kubeApiUrl, opts.kubeConfigFile, opts.kubeAnnotationPrefix)
}

cfg := &etcdv2.EtcdConfig{
Expand Down
66 changes: 66 additions & 0 deletions subnet/kube/annotations.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kube

import (
"errors"
"regexp"
"strings"
)

type annotations struct {
SubnetKubeManaged string
BackendData string
BackendType string
BackendPublicIP string
BackendPublicIPOverwrite string
}

func newAnnotations(prefix string) (annotations, error) {
slashCnt := strings.Count(prefix, "/")
if slashCnt > 1 {
return annotations{}, errors.New("subnet/kube: prefix can contain at most single slash")
}
if slashCnt == 0 {
prefix += "/"
}
if !strings.HasSuffix(prefix, "/") && !strings.HasSuffix(prefix, "-") {
prefix += "-"
}

// matches is a regexp matching the format used by the kubernetes for
// annotations. Following rules apply:
//
// - must start with FQDN - must contain at most one slash "/"
// - must contain only lowercase letters, nubers, underscores,
// hyphens, dots and slash
matches, err := regexp.MatchString(`(?:[a-z0-9_-]+\.)+[a-z0-9_-]+/(?:[a-z0-9_-]+-)?$`, prefix)
if err != nil {
panic(err)
}
if !matches {
return annotations{}, errors.New("subnet/kube: prefix must be in a format: fqdn/[0-9a-z-_]*")
}

a := annotations{
SubnetKubeManaged: prefix + "kube-subnet-manager",
BackendData: prefix + "backend-data",
BackendType: prefix + "backend-type",
BackendPublicIP: prefix + "public-ip",
BackendPublicIPOverwrite: prefix + "public-ip-overwrite",
}

return a, nil
}
96 changes: 96 additions & 0 deletions subnet/kube/annotations_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Copyright 2018 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kube

import "testing"

func Test_newAnnotations(t *testing.T) {
testCases := []struct {
prefix string
expectedBackendType string
hasError bool
}{
{
prefix: "flannel.alpha.coreos.com",
expectedBackendType: "flannel.alpha.coreos.com/backend-type",
},
{
prefix: "flannel.alpha.coreos.com/",
expectedBackendType: "flannel.alpha.coreos.com/backend-type",
},
{
prefix: "flannel.alpha.coreos.com/prefix",
expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type",
},
{
prefix: "flannel.alpha.coreos.com/prefix-",
expectedBackendType: "flannel.alpha.coreos.com/prefix-backend-type",
},
{
prefix: "org.com",
expectedBackendType: "org.com/backend-type",
},
{
prefix: "org9.com",
expectedBackendType: "org9.com/backend-type",
},
{
prefix: "org.com/9",
expectedBackendType: "org.com/9-backend-type",
},
{
// Not a fqdn.
prefix: "org",
hasError: true,
},
{
// Not a fqdn before /.
prefix: "org/",
hasError: true,
},
{
// Not a fqdn before /.
prefix: "org/prefix",
hasError: true,
},
{
// Uppercase letters.
prefix: "org.COM",
hasError: true,
},
{
// Uppercase letters.
prefix: "org.com/PREFIX",
hasError: true,
},
}

for i, tc := range testCases {
as, err := newAnnotations(tc.prefix)

if err != nil && !tc.hasError {
t.Errorf("#%d: error = %s, want nil", i, err)
continue
}
if err == nil && tc.hasError {
t.Errorf("#%d: error = nil, want non-nil", i)
continue
}

if as.BackendType != tc.expectedBackendType {
t.Errorf("#%d: BackendType = %s, want %s", i, as.BackendType, tc.expectedBackendType)
}
}
}
68 changes: 34 additions & 34 deletions subnet/kube/kube.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,11 @@ const (
resyncPeriod = 5 * time.Minute
nodeControllerSyncTimeout = 10 * time.Minute

subnetKubeManagedAnnotation = "flannel.alpha.coreos.com/kube-subnet-manager"
backendDataAnnotation = "flannel.alpha.coreos.com/backend-data"
backendTypeAnnotation = "flannel.alpha.coreos.com/backend-type"
backendPublicIPAnnotation = "flannel.alpha.coreos.com/public-ip"
backendPublicIPOverwriteAnnotation = "flannel.alpha.coreos.com/public-ip-overwrite"

netConfPath = "/etc/kube-flannel/net-conf.json"
)

type kubeSubnetManager struct {
annotations annotations
client clientset.Interface
nodeName string
nodeStore listers.NodeLister
Expand All @@ -69,7 +64,7 @@ type kubeSubnetManager struct {
events chan subnet.Event
}

func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) {
func NewSubnetManager(apiUrl, kubeconfig, prefix string) (subnet.Manager, error) {

var cfg *rest.Config
var err error
Expand Down Expand Up @@ -122,7 +117,7 @@ func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) {
return nil, fmt.Errorf("error parsing subnet config: %s", err)
}

sm, err := newKubeSubnetManager(c, sc, nodeName)
sm, err := newKubeSubnetManager(c, sc, nodeName, prefix)
if err != nil {
return nil, fmt.Errorf("error creating network manager: %s", err)
}
Expand All @@ -140,8 +135,13 @@ func NewSubnetManager(apiUrl, kubeconfig string) (subnet.Manager, error) {
return sm, nil
}

func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName string) (*kubeSubnetManager, error) {
func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName, prefix string) (*kubeSubnetManager, error) {
var err error
var ksm kubeSubnetManager
ksm.annotations, err = newAnnotations(prefix)
if err != nil {
return nil, err
}
ksm.client = c
ksm.nodeName = nodeName
ksm.subnetConf = sc
Expand Down Expand Up @@ -175,11 +175,11 @@ func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName str

func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj interface{}) {
n := obj.(*v1.Node)
if s, ok := n.Annotations[subnetKubeManagedAnnotation]; !ok || s != "true" {
if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
return
}

l, err := nodeToLease(*n)
l, err := ksm.nodeToLease(*n)
if err != nil {
glog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
return
Expand All @@ -190,16 +190,16 @@ func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj inter
func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{}) {
o := oldObj.(*v1.Node)
n := newObj.(*v1.Node)
if s, ok := n.Annotations[subnetKubeManagedAnnotation]; !ok || s != "true" {
if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
return
}
if o.Annotations[backendDataAnnotation] == n.Annotations[backendDataAnnotation] &&
o.Annotations[backendTypeAnnotation] == n.Annotations[backendTypeAnnotation] &&
o.Annotations[backendPublicIPAnnotation] == n.Annotations[backendPublicIPAnnotation] {
if o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
o.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] {
return // No change to lease
}

l, err := nodeToLease(*n)
l, err := ksm.nodeToLease(*n)
if err != nil {
glog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
return
Expand Down Expand Up @@ -233,24 +233,24 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
if err != nil {
return nil, err
}
if n.Annotations[backendDataAnnotation] != string(bd) ||
n.Annotations[backendTypeAnnotation] != attrs.BackendType ||
n.Annotations[backendPublicIPAnnotation] != attrs.PublicIP.String() ||
n.Annotations[subnetKubeManagedAnnotation] != "true" ||
(n.Annotations[backendPublicIPOverwriteAnnotation] != "" && n.Annotations[backendPublicIPOverwriteAnnotation] != attrs.PublicIP.String()) {
n.Annotations[backendTypeAnnotation] = attrs.BackendType
n.Annotations[backendDataAnnotation] = string(bd)
if n.Annotations[backendPublicIPOverwriteAnnotation] != "" {
if n.Annotations[backendPublicIPAnnotation] != n.Annotations[backendPublicIPOverwriteAnnotation] {
if n.Annotations[ksm.annotations.BackendData] != string(bd) ||
n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String()) {
n.Annotations[ksm.annotations.BackendType] = attrs.BackendType
n.Annotations[ksm.annotations.BackendData] = string(bd)
if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
glog.Infof("Overriding public ip with '%s' from node annotation '%s'",
n.Annotations[backendPublicIPOverwriteAnnotation],
backendPublicIPOverwriteAnnotation)
n.Annotations[backendPublicIPAnnotation] = n.Annotations[backendPublicIPOverwriteAnnotation]
n.Annotations[ksm.annotations.BackendPublicIPOverwrite],
ksm.annotations.BackendPublicIPOverwrite)
n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]
}
} else {
n.Annotations[backendPublicIPAnnotation] = attrs.PublicIP.String()
n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()
}
n.Annotations[subnetKubeManagedAnnotation] = "true"
n.Annotations[ksm.annotations.SubnetKubeManaged] = "true"

oldData, err := json.Marshal(cachedNode)
if err != nil {
Expand Down Expand Up @@ -295,14 +295,14 @@ func (ksm *kubeSubnetManager) Run(ctx context.Context) {
ksm.nodeController.Run(ctx.Done())
}

func nodeToLease(n v1.Node) (l subnet.Lease, err error) {
l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[backendPublicIPAnnotation])
func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error) {
l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])
if err != nil {
return l, err
}

l.Attrs.BackendType = n.Annotations[backendTypeAnnotation]
l.Attrs.BackendData = json.RawMessage(n.Annotations[backendDataAnnotation])
l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])

_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
if err != nil {
Expand Down