diff --git a/pkg/cli/admin/release/extract.go b/pkg/cli/admin/release/extract.go index 070535643e..4a3c242361 100644 --- a/pkg/cli/admin/release/extract.go +++ b/pkg/cli/admin/release/extract.go @@ -18,14 +18,18 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/yaml" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/cli-runtime/pkg/genericiooptions" "k8s.io/client-go/rest" kcmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" + configv1 "github.com/openshift/api/config/v1" imagev1 "github.com/openshift/api/image/v1" + configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" "github.com/openshift/library-go/pkg/image/dockerv1client" "github.com/openshift/library-go/pkg/manifest" "github.com/openshift/oc/pkg/cli/image/extract" @@ -349,17 +353,16 @@ func (o *ExtractOptions) Run(ctx context.Context) error { } } - tarEntryCallbacks := []extract.TarEntryFunc{} + var manifestsCallbacks []func(string, []manifest.Manifest, io.Reader, []configv1.ClusterVersionCapability) (bool, error) - var manifestErrs []error + var needEnabledCapabilities bool + inclusionConfig := manifestInclusionConfiguration{} if o.ExtractManifests { expectedProviderSpecKind := credRequestCloudProviderSpecKindMapping[o.Cloud] - - include := func(m *manifest.Manifest) error { return nil } // default to including everything if o.Included { context := "connected cluster" - inclusionConfig := manifestInclusionConfiguration{} if o.InstallConfig == "" { + needEnabledCapabilities = true inclusionConfig, err = findClusterIncludeConfig(ctx, o.RESTConfig) } else { inclusionConfig, err = findClusterIncludeConfigFromInstallConfig(ctx, o.InstallConfig) @@ -377,11 +380,10 @@ func (o *ExtractOptions) Run(ctx context.Context) error { return fmt.Errorf("unrecognized platform for CredentialsRequests: %q", *inclusionConfig.Platform) } } - include = newIncluder(inclusionConfig) } - tarEntryCallbacks = append(tarEntryCallbacks, func(hdr *tar.Header, _ extract.LayerInfo, r io.Reader) (bool, error) { - if hdr.Name == "image-references" && !o.CredentialsRequests { + manifestsCallbacks = append(manifestsCallbacks, func(name string, ms []manifest.Manifest, r io.Reader, enabled []configv1.ClusterVersionCapability) (cont bool, err error) { + if name == "image-references" && !o.CredentialsRequests { buf := &bytes.Buffer{} if _, err := io.Copy(buf, r); err != nil { return false, fmt.Errorf("unable to load image-references from release payload: %w", err) @@ -399,7 +401,7 @@ func (o *ExtractOptions) Run(ctx context.Context) error { out := o.Out if o.Directory != "" { - out, err = os.Create(filepath.Join(o.Directory, hdr.Name)) + out, err = os.Create(filepath.Join(o.Directory, name)) if err != nil { return false, err } @@ -409,10 +411,10 @@ func (o *ExtractOptions) Run(ctx context.Context) error { return true, err } return true, nil - } else if hdr.Name == "release-metadata" && !o.CredentialsRequests { + } else if name == "release-metadata" && !o.CredentialsRequests { out := o.Out if o.Directory != "" { - out, err = os.Create(filepath.Join(o.Directory, hdr.Name)) + out, err = os.Create(filepath.Join(o.Directory, name)) if err != nil { return false, err } @@ -424,22 +426,18 @@ func (o *ExtractOptions) Run(ctx context.Context) error { return true, nil } - if ext := path.Ext(hdr.Name); len(ext) == 0 || !(ext == ".yaml" || ext == ".yml" || ext == ".json") { - return true, nil - } - klog.V(4).Infof("Found manifest %s", hdr.Name) - ms, err := manifest.ParseManifests(r) - if err != nil { - manifestErrs = append(manifestErrs, errors.Wrapf(err, "error parsing %s", hdr.Name)) - return true, nil - } - for i := len(ms) - 1; i >= 0; i-- { if o.Included && o.CredentialsRequests && ms[i].GVK == credentialsRequestGVK && len(ms[i].Obj.GetAnnotations()) == 0 { klog.V(4).Infof("Including %s for manual CredentialsRequests, despite lack of annotations", ms[i].String()) - } else if err := include(&ms[i]); err != nil { - klog.V(4).Infof("Excluding %s: %s", ms[i].String(), err) - ms = append(ms[:i], ms[i+1:]...) + } else if o.Included { + clusterVersionCapabilitiesStatus := &configv1.ClusterVersionCapabilitiesStatus{ + KnownCapabilities: sets.New[configv1.ClusterVersionCapability](append(inclusionConfig.Capabilities.KnownCapabilities, configv1.KnownClusterVersionCapabilities...)...).UnsortedList(), + EnabledCapabilities: sets.New[configv1.ClusterVersionCapability](append(inclusionConfig.Capabilities.EnabledCapabilities, enabled...)...).UnsortedList(), + } + if err := ms[i].Include(inclusionConfig.ExcludeIdentifier, inclusionConfig.RequiredFeatureSet, inclusionConfig.Profile, clusterVersionCapabilitiesStatus, inclusionConfig.Overrides); err != nil { + klog.V(4).Infof("Excluding %s: %s", ms[i].String(), err) + ms = append(ms[:i], ms[i+1:]...) + } } } @@ -470,20 +468,20 @@ func (o *ExtractOptions) Run(ctx context.Context) error { out := o.Out if o.Directory != "" { - out, err = os.Create(filepath.Join(o.Directory, hdr.Name)) + out, err = os.Create(filepath.Join(o.Directory, name)) if err != nil { - return false, errors.Wrapf(err, "error creating manifest in %s", hdr.Name) + return false, errors.Wrapf(err, "error creating manifest in %s", name) } } if out != nil { for _, m := range manifestsToWrite { yamlBytes, err := yaml.JSONToYAML(m.Raw) if err != nil { - return false, errors.Wrapf(err, "error serializing manifest in %s", hdr.Name) + return false, errors.Wrapf(err, "error serializing manifest in %s", name) } fmt.Fprintf(out, "---\n") if _, err := out.Write(yamlBytes); err != nil { - return false, errors.Wrapf(err, "error writing manifest in %s", hdr.Name) + return false, errors.Wrapf(err, "error writing manifest in %s", name) } } } @@ -493,8 +491,8 @@ func (o *ExtractOptions) Run(ctx context.Context) error { fileFound := false if o.File != "" { - tarEntryCallbacks = append(tarEntryCallbacks, func(hdr *tar.Header, _ extract.LayerInfo, r io.Reader) (bool, error) { - if hdr.Name != o.File { + manifestsCallbacks = append(manifestsCallbacks, func(name string, _ []manifest.Manifest, r io.Reader, _ []configv1.ClusterVersionCapability) (bool, error) { + if name != o.File { return true, nil } fileFound = true @@ -502,22 +500,21 @@ func (o *ExtractOptions) Run(ctx context.Context) error { return false, err }) } - - if len(tarEntryCallbacks) > 0 { - tarEntryCallbacksDone := make([]bool, len(tarEntryCallbacks)) - opts.TarEntryCallback = func(hdr *tar.Header, layer extract.LayerInfo, r io.Reader) (bool, error) { - for i, callback := range tarEntryCallbacks { - if tarEntryCallbacksDone[i] { + manifestsCallback := func(name string, ms []manifest.Manifest, r io.Reader, enabled []configv1.ClusterVersionCapability) (bool, error) { + if len(manifestsCallbacks) > 0 { + callbacksDone := make([]bool, len(manifestsCallbacks)) + for i, callback := range manifestsCallbacks { + if callbacksDone[i] { continue } - if cont, err := callback(hdr, layer, r); err != nil { + if cont, err := callback(name, ms, r, enabled); err != nil { return cont, err } else if !cont { - tarEntryCallbacksDone[i] = true + callbacksDone[i] = true } } - for _, done := range tarEntryCallbacksDone { + for _, done := range callbacksDone { if !done { return true, nil // still some callbacks that want to keep working } @@ -525,7 +522,41 @@ func (o *ExtractOptions) Run(ctx context.Context) error { return false, nil } + return true, nil + } + + var currentPayloadManifests []manifest.Manifest + if needEnabledCapabilities { + optsToGetCurrentPayloadManifests, err := getOptsToGetCurrentPayloadManifests(ctx, opts, o.RESTConfig, inclusionConfig) + if err != nil { + return fmt.Errorf("error getting opts to get current payload manifests: %w", err) + } + optsToGetCurrentPayloadManifests.TarEntryCallback = func(h *tar.Header, _ extract.LayerInfo, r io.Reader) (cont bool, err error) { + if ext := path.Ext(h.Name); len(ext) == 0 || !(ext == ".yaml" || ext == ".yml" || ext == ".json") { + return true, nil + } + klog.V(4).Infof("Found manifest %s in the current release payload", h.Name) + ms, err := manifest.ParseManifests(r) + if err != nil { + return false, err + } + for i := len(ms) - 1; i >= 0; i-- { + if err := ms[i].Include(inclusionConfig.ExcludeIdentifier, inclusionConfig.RequiredFeatureSet, inclusionConfig.Profile, inclusionConfig.Capabilities, inclusionConfig.Overrides); err != nil { + klog.V(4).Infof("Excluding %s in the current release payload: %s", ms[i].String(), err) + ms = append(ms[:i], ms[i+1:]...) + } + } + currentPayloadManifests = append(currentPayloadManifests, ms...) + return true, nil + } + if err := optsToGetCurrentPayloadManifests.Run(); err != nil { + return fmt.Errorf("error getting current payload manifests: %w", err) + } + } + manifestReceiver := NewManifestReceiver(manifestsCallback, needEnabledCapabilities, sets.New[string]("image-references", "release-metadata"), currentPayloadManifests, inclusionConfig) + opts.TarEntryCallback = manifestReceiver.TarEntryCallback + opts.TarEntryCallbackDoneCallback = manifestReceiver.TarEntryCallbackDoneCallback if err := opts.Run(); err != nil { return err @@ -553,14 +584,48 @@ func (o *ExtractOptions) Run(ctx context.Context) error { // Only output manifest errors if manifests were being extracted. // Do not return an error so current operation, e.g. mirroring, continues. - if o.ExtractManifests && len(manifestErrs) > 0 { - fmt.Fprintf(o.ErrOut, "Errors: %s\n", errorList(manifestErrs)) + if o.ExtractManifests && len(manifestReceiver.ManifestErrs) > 0 { + fmt.Fprintf(o.ErrOut, "Errors: %s\n", errorList(manifestReceiver.ManifestErrs)) } return nil } +func getOptsToGetCurrentPayloadManifests(ctx context.Context, source *extract.ExtractOptions, config *rest.Config, inclusionConfiguration manifestInclusionConfiguration) (*extract.ExtractOptions, error) { + client, err := configv1client.NewForConfig(config) + if err != nil { + return nil, err + } + + clusterVersion, err := client.ClusterVersions().Get(ctx, "version", metav1.GetOptions{}) + if err != nil { + return nil, err + } + + src := clusterVersion.Status.Desired.Image + ref, err := imagesource.ParseReference(src) + if err != nil { + return nil, err + } + klog.V(4).Infof("The outgoing release payload from %s is running on the cluster: %s", src, config.Host) + opts := extract.NewExtractOptions(genericiooptions.IOStreams{Out: source.Out, ErrOut: source.ErrOut}) + opts.ParallelOptions = source.ParallelOptions + opts.SecurityOptions = source.SecurityOptions + opts.FilterOptions = source.FilterOptions + opts.FileDir = source.FileDir + opts.OnlyFiles = true + opts.ICSPFile = source.ICSPFile + opts.IDMSFile = source.IDMSFile + opts.Mappings = []extract.Mapping{ + { + ImageRef: ref, + From: "release-manifests/", + }, + } + return opts, nil +} + func (o *ExtractOptions) extractGit(dir string) error { switch o.Output { case "commit", "": diff --git a/pkg/cli/admin/release/extract_tools.go b/pkg/cli/admin/release/extract_tools.go index bfb46da691..446d86ff00 100644 --- a/pkg/cli/admin/release/extract_tools.go +++ b/pkg/cli/admin/release/extract_tools.go @@ -13,8 +13,8 @@ import ( "hash" "io" "os" + "path" "path/filepath" - "regexp" "runtime" "sort" "strings" @@ -44,6 +44,7 @@ import ( "github.com/openshift/oc/pkg/cli/image/extract" "github.com/openshift/oc/pkg/cli/image/imagesource" "github.com/openshift/oc/pkg/version" + "github.com/pkg/errors" ) // extractTarget describes how a file in the release image can be extracted to disk. @@ -1167,6 +1168,106 @@ func copyAndReplace(errorOutput io.Writer, w io.Writer, r io.Reader, bufferSize } +// ManifestReceiver accepts TarEntryCallback calls from the upstream, parses the manifests from the input there, +// and feeds its manifestsCallback with them. +// If needEnabledCapabilities is not set, manifestsCallback is called after each TarEntryCallback call. In this case, the +// argument enabledCapabilities is nil in the manifestsCallback. +// If waitAll is set, manifestsCallback is called only after all TarEntryCallback calls have been received, i.e., +// all the manifests have been handled. The manifests will be used as the payloads of a release to update a cluster to +// calculate the enabled capabilities after the update with the given manifestInclusionConfiguration. The enabled +// capabilities is sent to manifestsCallback. +// The string set skipNames contains the file names that should not be parsed manifests and keep the reader from the +// upstream intact so that manifestsCallback can read the content from it. +type ManifestReceiver struct { + manifestsCallback func(filename string, manifests []manifest.Manifest, reader io.Reader, enabledCapabilities []configv1.ClusterVersionCapability) (cont bool, err error) + needEnabledCapabilities bool + skipNames sets.Set[string] + currentPayloadManifests []manifest.Manifest + inclusionConfiguration manifestInclusionConfiguration + + cache []cacheData + manifests []manifest.Manifest + enabledResolved bool + enabled []configv1.ClusterVersionCapability + + ManifestErrs []error +} + +type cacheData struct { + name string + ms []manifest.Manifest + reader io.Reader +} + +func NewManifestReceiver(tarEntryCallback func(string, []manifest.Manifest, io.Reader, []configv1.ClusterVersionCapability) (cont bool, err error), needEnabledCapabilities bool, skipNames sets.Set[string], currentPayloadManifests []manifest.Manifest, inclusionConfiguration manifestInclusionConfiguration) *ManifestReceiver { + return &ManifestReceiver{manifestsCallback: tarEntryCallback, needEnabledCapabilities: needEnabledCapabilities, skipNames: skipNames, currentPayloadManifests: currentPayloadManifests, inclusionConfiguration: inclusionConfiguration} +} + +func (mr *ManifestReceiver) TarEntryCallback(h *tar.Header, _ extract.LayerInfo, r io.Reader) (cont bool, err error) { + data, err := io.ReadAll(r) + if err != nil { + return false, err + } + buf := bytes.NewBuffer(data) + if mr.skipNames.Has(h.Name) { + mr.cache = append(mr.cache, cacheData{name: h.Name, reader: buf}) + return true, nil + } + + if ext := path.Ext(h.Name); len(ext) == 0 || !(ext == ".yaml" || ext == ".yml" || ext == ".json") { + return true, nil + } + klog.V(4).Infof("Found manifest %s", h.Name) + ms, err := manifest.ParseManifests(buf) + if err != nil { + mr.ManifestErrs = append(mr.ManifestErrs, errors.Wrapf(err, "error parsing %s", h.Name)) + return true, nil + } + mr.manifests = append(mr.manifests, ms...) + + if mr.needEnabledCapabilities { + mr.cache = append(mr.cache, cacheData{name: h.Name, ms: ms}) + return true, nil + } + return mr.manifestsCallback(h.Name, ms, nil, nil) +} + +func (mr *ManifestReceiver) TarEntryCallbackDoneCallback() error { + defer func() { + mr.cache = []cacheData{} + }() + + if mr.needEnabledCapabilities && !mr.enabledResolved { + enabled := GetImplicitlyEnabledCapabilities( + mr.manifests, + mr.currentPayloadManifests, + mr.inclusionConfiguration, + sets.New[configv1.ClusterVersionCapability](), + ) + + delta := enabled.Clone() + if mr.inclusionConfiguration.Capabilities != nil { + delta = delta.Difference(sets.New[configv1.ClusterVersionCapability](mr.inclusionConfiguration.Capabilities.EnabledCapabilities...)) + enabled.Insert(mr.inclusionConfiguration.Capabilities.EnabledCapabilities...) + } + klog.Infof("Those capabilities become implicitly enabled for the incoming release %s", delta.UnsortedList()) + + mr.enabled = enabled.UnsortedList() + mr.enabledResolved = true + } + + for _, c := range mr.cache { + cont, err := mr.manifestsCallback(c.name, c.ms, c.reader, mr.enabled) + if err != nil { + return err + } + if !cont { + return nil + } + } + return nil +} + func findClusterIncludeConfigFromInstallConfig(ctx context.Context, installConfigPath string) (manifestInclusionConfiguration, error) { config := manifestInclusionConfiguration{} @@ -1237,19 +1338,6 @@ func findClusterIncludeConfig(ctx context.Context, restConfig *rest.Config) (man } else { config.Overrides = clusterVersion.Spec.Overrides config.Capabilities = &clusterVersion.Status.Capabilities - - // FIXME: eventually pull in GetImplicitlyEnabledCapabilities from /~https://github.com/openshift/cluster-version-operator/blob/86e24d66119a73f50282b66a8d6f2e3518aa0e15/pkg/payload/payload.go#L237-L240 for cases where a minor update would implicitly enable some additional capabilities. For now, 4.13 to 4.14 will always enable MachineAPI, ImageRegistry, etc.. - currentVersion := clusterVersion.Status.Desired.Version - matches := regexp.MustCompile(`^(\d+[.]\d+)[.].*`).FindStringSubmatch(currentVersion) - if len(matches) < 2 { - return config, fmt.Errorf("failed to parse major.minor version from ClusterVersion status.desired.version %q", currentVersion) - } else if matches[1] == "4.13" { - build := configv1.ClusterVersionCapability("Build") - deploymentConfig := configv1.ClusterVersionCapability("DeploymentConfig") - imageRegistry := configv1.ClusterVersionCapability("ImageRegistry") - config.Capabilities.EnabledCapabilities = append(config.Capabilities.EnabledCapabilities, configv1.ClusterVersionCapabilityMachineAPI, build, deploymentConfig, imageRegistry) - config.Capabilities.KnownCapabilities = append(config.Capabilities.KnownCapabilities, configv1.ClusterVersionCapabilityMachineAPI, build, deploymentConfig, imageRegistry) - } } if infrastructure, err := client.Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}); err != nil { @@ -1281,8 +1369,59 @@ func findClusterIncludeConfig(ctx context.Context, restConfig *rest.Config) (man return config, nil } -func newIncluder(config manifestInclusionConfiguration) includer { - return func(m *manifest.Manifest) error { - return m.Include(config.ExcludeIdentifier, config.RequiredFeatureSet, config.Profile, config.Capabilities, config.Overrides) +// GetImplicitlyEnabledCapabilities returns a set of capabilities that are implicitly enabled after a cluster update. +// The arguments are two sets of manifests, manifest inclusion configuration, and +// a set of capabilities that are implicitly enabled on the cluster, i.e., the capabilities +// that are NOT specified in the cluster version but has to considered enabled on the cluster. +// The manifest inclusion configuration is used to determine if a manifest should be included. +// In other words, whether, or not the cluster version operator reconcile that manifest on the cluster. +// The two sets of manifests are respectively from the release that is currently running on the cluster and +// from the release that the cluster is updated to. +// TODO lift this function to library-go +func GetImplicitlyEnabledCapabilities( + updatePayloadManifests []manifest.Manifest, + currentPayloadManifests []manifest.Manifest, + manifestInclusionConfiguration manifestInclusionConfiguration, + currentImplicitlyEnabled sets.Set[configv1.ClusterVersionCapability], +) sets.Set[configv1.ClusterVersionCapability] { + + ret := sets.New[configv1.ClusterVersionCapability]().Union(currentImplicitlyEnabled) + for _, updateManifest := range updatePayloadManifests { + updateManErr := updateManifest.IncludeAllowUnknownCapabilities( + manifestInclusionConfiguration.ExcludeIdentifier, + manifestInclusionConfiguration.RequiredFeatureSet, + manifestInclusionConfiguration.Profile, + manifestInclusionConfiguration.Capabilities, + manifestInclusionConfiguration.Overrides, + true, + ) + // update manifest is enabled, no need to check + if updateManErr == nil { + continue + } + for _, currentManifest := range currentPayloadManifests { + if !updateManifest.SameResourceID(currentManifest) { + continue + } + // current manifest is disabled, no need to check + if err := currentManifest.IncludeAllowUnknownCapabilities( + manifestInclusionConfiguration.ExcludeIdentifier, + manifestInclusionConfiguration.RequiredFeatureSet, + manifestInclusionConfiguration.Profile, + manifestInclusionConfiguration.Capabilities, + manifestInclusionConfiguration.Overrides, + true, + ); err != nil { + continue + } + newImplicitlyEnabled := sets.New[configv1.ClusterVersionCapability](updateManifest.GetManifestCapabilities()...). + Difference(sets.New[configv1.ClusterVersionCapability](currentManifest.GetManifestCapabilities()...)). + Difference(currentImplicitlyEnabled). + Difference(sets.New[configv1.ClusterVersionCapability](manifestInclusionConfiguration.Capabilities.EnabledCapabilities...)) + ret = ret.Union(newImplicitlyEnabled) + klog.V(2).Infof("%s has changed and is now part of one or more disabled capabilities. The following capabilities will be implicitly enabled: %s", + updateManifest.String(), newImplicitlyEnabled.UnsortedList()) + } } + return ret } diff --git a/pkg/cli/image/extract/extract.go b/pkg/cli/image/extract/extract.go index 0285388710..518b010de9 100644 --- a/pkg/cli/image/extract/extract.go +++ b/pkg/cli/image/extract/extract.go @@ -148,6 +148,9 @@ type ExtractOptions struct { // by name and only the entry in the highest layer will be passed to the callback. Returning false // will halt processing of the image. TarEntryCallback TarEntryFunc + // TarEntryCallbackDoneCallback, if set, is called when all layers image have been handled, i.e., no more + // TarEntryCallback is going to be passed. It has no effect if TarEntryCallback is not set. + TarEntryCallbackDoneCallback func() error // AllLayers ensures the TarEntryCallback is invoked for all files, and will cause the callback // order to start at the lowest layer and work outwards. AllLayers bool @@ -544,6 +547,12 @@ func (o *ExtractOptions) Run() error { } } + if o.TarEntryCallback != nil && o.TarEntryCallbackDoneCallback != nil { + if err := o.TarEntryCallbackDoneCallback(); err != nil { + return err + } + } + if o.ImageMetadataCallback != nil { o.ImageMetadataCallback(&mapping, location.Manifest, contentDigest, imageConfig, location.ManifestListDigest()) }