Merge pull request kubernetes#24150 from deads2k/dynamic-cert-pick-01

dynamic certs and cluster auth

cmd/kube-apiserver/app/server.go

@@ -521,7 +521,7 @@ func buildGenericConfig(
 	}
 	versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
 
-	genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, genericConfig.Authentication.DynamicReloadFns, err = BuildAuthenticator(s, clientgoExternalClient, versionedInformers)
+	genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, clientgoExternalClient, versionedInformers)
 	if err != nil {
 		lastErr = fmt.Errorf("invalid authentication config: %v", err)
 		return
@@ -587,8 +587,11 @@ func buildGenericConfig(
 }
 
 // BuildAuthenticator constructs the authenticator
-func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, map[string]genericapiserver.PostStartHookFunc, error) {
-	authenticatorConfig := s.Authentication.ToAuthenticationConfig()
+func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
+	authenticatorConfig, err := s.Authentication.ToAuthenticationConfig()
+	if err != nil {
+		return nil, nil, err
+	}
 	if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
 		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
 			extclient,

cmd/kubelet/app/auth.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	clientset "k8s.io/client-go/kubernetes"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1beta1"
@@ -63,10 +64,19 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
 
 // BuildAuthn creates an authenticator compatible with the kubelet's needs
 func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
+	var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider
+	var err error
+	if len(authn.X509.ClientCAFile) > 0 {
+		clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous:    authn.Anonymous.Enabled,
-		CacheTTL:     authn.Webhook.CacheTTL.Duration,
-		ClientCAFile: authn.X509.ClientCAFile,
+		Anonymous:                          authn.Anonymous.Enabled,
+		CacheTTL:                           authn.Webhook.CacheTTL.Duration,
+		ClientCertificateCAContentProvider: clientCertificateCAContentProvider,
 	}
 
 	if authn.Webhook.Enabled {
@@ -76,8 +86,7 @@ func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletc
 		authenticatorConfig.TokenAccessReviewClient = client
 	}
 
-	// ignore dynamic reload.  We may want this in the future
-	authenticator, _, _, err := authenticatorConfig.New()
+	authenticator, _, err := authenticatorConfig.New()
 	return authenticator, err
 }
 

openshift-kube-apiserver/openshiftkubeapiserver/patch_authenticator.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/group"
 	"k8s.io/apiserver/pkg/authentication/request/anonymous"
@@ -16,7 +18,6 @@ import (
 	tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
 	tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
 	genericapiserver "k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/server/certs"
 	webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 	kclientsetexternal "k8s.io/client-go/kubernetes"
 	corev1listers "k8s.io/client-go/listers/core/v1"
@@ -178,34 +179,42 @@ func newAuthenticator(
 	// build cert authenticator
 	// TODO: add "system:" prefix in authenticator, limit cert to username
 	// TODO: add "system:" prefix to groups in authenticator, limit cert to group name
-	dynamicCA := certs.NewDynamicCA(apiClientCABundle)
-	if err := dynamicCA.CheckCerts(); err != nil {
+	clientCAContentProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", apiClientCABundle)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := clientCAContentProvider.RunOnce(); err != nil {
 		return nil, nil, err
 	}
-	certauth := x509request.NewDynamic(dynamicCA.GetVerifier, x509request.CommonNameUserConversion)
+	certAuth := x509request.NewDynamic(clientCAContentProvider.VerifyOptions, x509request.CommonNameUserConversion)
 	postStartHooks["openshift.io-clientCA-reload"] = func(context genericapiserver.PostStartHookContext) error {
-		go dynamicCA.Run(context.StopCh)
+		go clientCAContentProvider.Run(1, context.StopCh)
 		return nil
 	}
-	authenticators = append(authenticators, certauth)
+	authenticators = append(authenticators, certAuth)
 
 	resultingAuthenticator := union.NewFailOnError(authenticators...)
 
 	topLevelAuthenticators := []authenticator.Request{}
 	// if we have a front proxy providing authentication configuration, wire it up and it should come first
 	if authConfig.RequestHeader != nil {
-		requestHeaderAuthenticator, dynamicReloadFn, err := headerrequest.NewSecure(
-			authConfig.RequestHeader.ClientCA,
-			authConfig.RequestHeader.ClientCommonNames,
-			authConfig.RequestHeader.UsernameHeaders,
-			authConfig.RequestHeader.GroupHeaders,
-			authConfig.RequestHeader.ExtraHeaderPrefixes,
-		)
+		caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", authConfig.RequestHeader.ClientCA)
 		if err != nil {
-			return nil, nil, fmt.Errorf("Error building front proxy auth config: %v", err)
+			return nil, nil, err
+		}
+		if err := caBundleProvider.RunOnce(); err != nil {
+			return nil, nil, err
 		}
+
+		requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
+			caBundleProvider.VerifyOptions,
+			headerrequest.StaticStringSlice(authConfig.RequestHeader.ClientCommonNames),
+			headerrequest.StaticStringSlice(authConfig.RequestHeader.UsernameHeaders),
+			headerrequest.StaticStringSlice(authConfig.RequestHeader.GroupHeaders),
+			headerrequest.StaticStringSlice(authConfig.RequestHeader.ExtraHeaderPrefixes),
+		)
 		postStartHooks["openshift.io-requestheader-reload"] = func(context genericapiserver.PostStartHookContext) error {
-			go dynamicReloadFn(context.StopCh)
+			go caBundleProvider.Run(1, context.StopCh)
 			return nil
 		}
 		topLevelAuthenticators = append(topLevelAuthenticators, union.New(requestHeaderAuthenticator, resultingAuthenticator))

pkg/kubeapiserver/authenticator/config.go

@@ -17,7 +17,6 @@ limitations under the License.
 package authenticator
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/go-openapi/spec"
@@ -34,8 +33,7 @@ import (
 	tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
-	genericapiserver "k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/server/certs"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile"
 	"k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth"
@@ -44,18 +42,17 @@ import (
 
 	// Initialize all known client auth plugins.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
 // Config contains the data on how to authenticate a request to the Kube API Server
 type Config struct {
-	Anonymous                   bool
-	BasicAuthFile               string
-	BootstrapToken              bool
-	ClientCAFile                string
+	Anonymous      bool
+	BasicAuthFile  string
+	BootstrapToken bool
+
 	TokenAuthFile               string
 	OIDCIssuerURL               string
 	OIDCClientID                string
@@ -71,6 +68,7 @@ type Config struct {
 	ServiceAccountIssuer        string
 	APIAudiences                authenticator.Audiences
 	WebhookTokenAuthnConfigFile string
+	WebhookTokenAuthnVersion    string
 	WebhookTokenAuthnCacheTTL   time.Duration
 
 	TokenSuccessCacheTTL time.Duration
@@ -81,41 +79,37 @@ type Config struct {
 	// TODO, this is the only non-serializable part of the entire config.  Factor it out into a clientconfig
 	ServiceAccountTokenGetter   serviceaccount.ServiceAccountTokenGetter
 	BootstrapTokenAuthenticator authenticator.Token
+	// ClientCAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// Generally this is the CA bundle file used to authenticate client certificates
+	// If this value is nil, then mutual TLS is disabled.
+	ClientCAContentProvider dynamiccertificates.CAContentProvider
 }
 
 // New returns an authenticator.Request or an error that supports the standard
 // Kubernetes authentication mechanisms.
-func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, map[string]genericapiserver.PostStartHookFunc, error) {
+func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
 	var authenticators []authenticator.Request
 	var tokenAuthenticators []authenticator.Token
 	securityDefinitions := spec.SecurityDefinitions{}
-	dynamicReloadHooks := map[string]genericapiserver.PostStartHookFunc{}
 
 	// front-proxy, BasicAuth methods, local first, then remote
 	// Add the front proxy authenticator if requested
 	if config.RequestHeaderConfig != nil {
-		requestHeaderAuthenticator, dynamicReloadFn, err := headerrequest.NewSecure(
-			config.RequestHeaderConfig.ClientCA,
+		requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
+			config.RequestHeaderConfig.CAContentProvider.VerifyOptions,
 			config.RequestHeaderConfig.AllowedClientNames,
 			config.RequestHeaderConfig.UsernameHeaders,
 			config.RequestHeaderConfig.GroupHeaders,
 			config.RequestHeaderConfig.ExtraHeaderPrefixes,
 		)
-		if err != nil {
-			return nil, nil, nil, err
-		}
-		dynamicReloadHooks["kube-apiserver-requestheader-reload"] = func(context genericapiserver.PostStartHookContext) error {
-			go dynamicReloadFn(context.StopCh)
-			return nil
-		}
 		authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, requestHeaderAuthenticator))
 	}
 
 	// basic auth
 	if len(config.BasicAuthFile) > 0 {
 		basicAuth, err := newAuthenticatorFromBasicAuthFile(config.BasicAuthFile)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, basicAuth))
 
@@ -128,37 +122,30 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, ma
 	}
 
 	// X509 methods
-	if len(config.ClientCAFile) > 0 {
-		dynamicVerifier := certs.NewDynamicCA(config.ClientCAFile)
-		if err := dynamicVerifier.CheckCerts(); err != nil {
-			return nil, nil, nil, fmt.Errorf("unable to load client CA file %s: %v", config.ClientCAFile, err)
-		}
-		dynamicReloadHooks["kube-apiserver-clientCA-reload"] = func(context genericapiserver.PostStartHookContext) error {
-			go dynamicVerifier.Run(context.StopCh)
-			return nil
-		}
-		authenticators = append(authenticators, x509.NewDynamic(dynamicVerifier.GetVerifier, x509.CommonNameUserConversion))
+	if config.ClientCAContentProvider != nil {
+		certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)
+		authenticators = append(authenticators, certAuth)
 	}
 
 	// Bearer token methods, local first, then remote
 	if len(config.TokenAuthFile) > 0 {
 		tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
 	}
 	if len(config.ServiceAccountKeyFiles) > 0 {
 		serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
 	}
 	if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) && config.ServiceAccountIssuer != "" {
 		serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuer, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
 	}
@@ -188,14 +175,14 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, ma
 			RequiredClaims:       config.OIDCRequiredClaims,
 		})
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, oidcAuth)
 	}
 	if len(config.WebhookTokenAuthnConfigFile) > 0 {
-		webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnCacheTTL, config.APIAudiences)
+		webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnVersion, config.WebhookTokenAuthnCacheTTL, config.APIAudiences)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
 	}
@@ -220,9 +207,9 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, ma
 
 	if len(authenticators) == 0 {
 		if config.Anonymous {
-			return anonymous.NewAuthenticator(), &securityDefinitions, dynamicReloadHooks, nil
+			return anonymous.NewAuthenticator(), &securityDefinitions, nil
 		}
-		return nil, &securityDefinitions, dynamicReloadHooks, nil
+		return nil, &securityDefinitions, nil
 	}
 
 	authenticator := union.New(authenticators...)
@@ -235,7 +222,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, ma
 		authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
 	}
 
-	return authenticator, &securityDefinitions, dynamicReloadHooks, nil
+	return authenticator, &securityDefinitions, nil
 }
 
 // IsValidServiceAccountKeyFile returns true if a valid public RSA key can be read from the given file
@@ -319,20 +306,7 @@ func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences
 	return tokenAuthenticator, nil
 }
 
-// newAuthenticatorFromClientCAFile returns an authenticator.Request or an error
-func newAuthenticatorFromClientCAFile(clientCAFile string) (authenticator.Request, error) {
-	roots, err := certutil.NewPool(clientCAFile)
-	if err != nil {
-		return nil, err
-	}
-
-	opts := x509.DefaultVerifyOptions()
-	opts.Roots = roots
-
-	return x509.New(opts, x509.CommonNameUserConversion), nil
-}
-
-func newWebhookTokenAuthenticator(webhookConfigFile string, ttl time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) {
+func newWebhookTokenAuthenticator(webhookConfigFile string, version string, ttl time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) {
 	webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, implicitAuds)
 	if err != nil {
 		return nil, err