Skip to content

Commit

Permalink
OTA-1010: extract included manifests with net-new capabilities
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongkailiu
hongkailiu committed Jan 14, 2025
1 parent e3b2070 commit 38aeb1d
Show file tree
Hide file tree
Showing 3 changed files with 211 additions and 57 deletions.
78 changes: 38 additions & 40 deletions pkg/cli/admin/release/extract.go
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
package release

import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"path"
"path/filepath"
"sync"
"time"
Expand All @@ -20,11 +18,13 @@ import (

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/cli-runtime/pkg/genericiooptions"
"k8s.io/client-go/rest"
kcmdutil "k8s.io/kubectl/pkg/cmd/util"
"k8s.io/kubectl/pkg/util/templates"

configv1 "github.com/openshift/api/config/v1"
imagev1 "github.com/openshift/api/image/v1"
"github.com/openshift/library-go/pkg/image/dockerv1client"
"github.com/openshift/library-go/pkg/manifest"
Expand Down Expand Up @@ -349,17 +349,16 @@ func (o *ExtractOptions) Run(ctx context.Context) error {
}
}

tarEntryCallbacks := []extract.TarEntryFunc{}
var manifestsCallbacks []func(string, []manifest.Manifest, io.Reader, []configv1.ClusterVersionCapability) (bool, error)

var manifestErrs []error
var needEnabledCapabilities bool
inclusionConfig := manifestInclusionConfiguration{}
if o.ExtractManifests {
expectedProviderSpecKind := credRequestCloudProviderSpecKindMapping[o.Cloud]

include := func(m *manifest.Manifest) error { return nil } // default to including everything
if o.Included {
context := "connected cluster"
inclusionConfig := manifestInclusionConfiguration{}
if o.InstallConfig == "" {
needEnabledCapabilities = true
inclusionConfig, err = findClusterIncludeConfig(ctx, o.RESTConfig)
} else {
inclusionConfig, err = findClusterIncludeConfigFromInstallConfig(ctx, o.InstallConfig)
Expand All @@ -377,11 +376,10 @@ func (o *ExtractOptions) Run(ctx context.Context) error {
return fmt.Errorf("unrecognized platform for CredentialsRequests: %q", *inclusionConfig.Platform)
}
}
include = newIncluder(inclusionConfig)
}

tarEntryCallbacks = append(tarEntryCallbacks, func(hdr *tar.Header, _ extract.LayerInfo, r io.Reader) (bool, error) {
if hdr.Name == "image-references" && !o.CredentialsRequests {
manifestsCallbacks = append(manifestsCallbacks, func(name string, ms []manifest.Manifest, r io.Reader, enabled []configv1.ClusterVersionCapability) (cont bool, err error) {
if name == "image-references" && !o.CredentialsRequests {
buf := &bytes.Buffer{}
if _, err := io.Copy(buf, r); err != nil {
return false, fmt.Errorf("unable to load image-references from release payload: %w", err)
Expand All @@ -399,7 +397,7 @@ func (o *ExtractOptions) Run(ctx context.Context) error {

out := o.Out
if o.Directory != "" {
out, err = os.Create(filepath.Join(o.Directory, hdr.Name))
out, err = os.Create(filepath.Join(o.Directory, name))
if err != nil {
return false, err
}
Expand All @@ -409,10 +407,10 @@ func (o *ExtractOptions) Run(ctx context.Context) error {
return true, err
}
return true, nil
} else if hdr.Name == "release-metadata" && !o.CredentialsRequests {
} else if name == "release-metadata" && !o.CredentialsRequests {
out := o.Out
if o.Directory != "" {
out, err = os.Create(filepath.Join(o.Directory, hdr.Name))
out, err = os.Create(filepath.Join(o.Directory, name))
if err != nil {
return false, err
}
Expand All @@ -424,22 +422,18 @@ func (o *ExtractOptions) Run(ctx context.Context) error {
return true, nil
}

if ext := path.Ext(hdr.Name); len(ext) == 0 || !(ext == ".yaml" || ext == ".yml" || ext == ".json") {
return true, nil
}
klog.V(4).Infof("Found manifest %s", hdr.Name)
ms, err := manifest.ParseManifests(r)
if err != nil {
manifestErrs = append(manifestErrs, errors.Wrapf(err, "error parsing %s", hdr.Name))
return true, nil
}

for i := len(ms) - 1; i >= 0; i-- {
if o.Included && o.CredentialsRequests && ms[i].GVK == credentialsRequestGVK && len(ms[i].Obj.GetAnnotations()) == 0 {
klog.V(4).Infof("Including %s for manual CredentialsRequests, despite lack of annotations", ms[i].String())
} else if err := include(&ms[i]); err != nil {
klog.V(4).Infof("Excluding %s: %s", ms[i].String(), err)
ms = append(ms[:i], ms[i+1:]...)
} else if o.Included {
clusterVersionCapabilitiesStatus := &configv1.ClusterVersionCapabilitiesStatus{
KnownCapabilities: append(inclusionConfig.Capabilities.KnownCapabilities, configv1.KnownClusterVersionCapabilities...),
EnabledCapabilities: append(inclusionConfig.Capabilities.EnabledCapabilities, enabled...),
}
if err := ms[i].Include(inclusionConfig.ExcludeIdentifier, inclusionConfig.RequiredFeatureSet, inclusionConfig.Profile, clusterVersionCapabilitiesStatus, inclusionConfig.Overrides); err != nil {
klog.V(4).Infof("Excluding %s: %s", ms[i].String(), err)
ms = append(ms[:i], ms[i+1:]...)
}
}
}

Expand Down Expand Up @@ -470,20 +464,20 @@ func (o *ExtractOptions) Run(ctx context.Context) error {

out := o.Out
if o.Directory != "" {
out, err = os.Create(filepath.Join(o.Directory, hdr.Name))
out, err = os.Create(filepath.Join(o.Directory, name))
if err != nil {
return false, errors.Wrapf(err, "error creating manifest in %s", hdr.Name)
return false, errors.Wrapf(err, "error creating manifest in %s", name)
}
}
if out != nil {
for _, m := range manifestsToWrite {
yamlBytes, err := yaml.JSONToYAML(m.Raw)
if err != nil {
return false, errors.Wrapf(err, "error serializing manifest in %s", hdr.Name)
return false, errors.Wrapf(err, "error serializing manifest in %s", name)
}
fmt.Fprintf(out, "---\n")
if _, err := out.Write(yamlBytes); err != nil {
return false, errors.Wrapf(err, "error writing manifest in %s", hdr.Name)
return false, errors.Wrapf(err, "error writing manifest in %s", name)
}
}
}
Expand All @@ -493,24 +487,23 @@ func (o *ExtractOptions) Run(ctx context.Context) error {

fileFound := false
if o.File != "" {
tarEntryCallbacks = append(tarEntryCallbacks, func(hdr *tar.Header, _ extract.LayerInfo, r io.Reader) (bool, error) {
if hdr.Name != o.File {
manifestsCallbacks = append(manifestsCallbacks, func(name string, _ []manifest.Manifest, r io.Reader, _ []configv1.ClusterVersionCapability) (bool, error) {
if name != o.File {
return true, nil
}
fileFound = true
_, err := io.Copy(o.Out, r)
return false, err
})
}

if len(tarEntryCallbacks) > 0 {
tarEntryCallbacksDone := make([]bool, len(tarEntryCallbacks))
opts.TarEntryCallback = func(hdr *tar.Header, layer extract.LayerInfo, r io.Reader) (bool, error) {
for i, callback := range tarEntryCallbacks {
manifestsCallback := func(name string, ms []manifest.Manifest, r io.Reader, enabled []configv1.ClusterVersionCapability) (bool, error) {
if len(manifestsCallbacks) > 0 {
tarEntryCallbacksDone := make([]bool, len(manifestsCallbacks))
for i, callback := range manifestsCallbacks {
if tarEntryCallbacksDone[i] {
continue
}
if cont, err := callback(hdr, layer, r); err != nil {
if cont, err := callback(name, ms, r, enabled); err != nil {
return cont, err
} else if !cont {
tarEntryCallbacksDone[i] = true
Expand All @@ -525,8 +518,13 @@ func (o *ExtractOptions) Run(ctx context.Context) error {

return false, nil
}
return true, nil
}

manifestReceiver := NewManifestReceiver(manifestsCallback, needEnabledCapabilities, sets.New[string]("image-references", "release-metadata"), inclusionConfig)
opts.TarEntryCallback = manifestReceiver.TarEntryCallback
opts.TarEntryCallbackDoneCallback = manifestReceiver.TarEntryCallbackDoneCallback

if err := opts.Run(); err != nil {
return err
}
Expand All @@ -553,8 +551,8 @@ func (o *ExtractOptions) Run(ctx context.Context) error {

// Only output manifest errors if manifests were being extracted.
// Do not return an error so current operation, e.g. mirroring, continues.
if o.ExtractManifests && len(manifestErrs) > 0 {
fmt.Fprintf(o.ErrOut, "Errors: %s\n", errorList(manifestErrs))
if o.ExtractManifests && len(manifestReceiver.ManifestErrs) > 0 {
fmt.Fprintf(o.ErrOut, "Errors: %s\n", errorList(manifestReceiver.ManifestErrs))
}

return nil
Expand Down
Loading

0 comments on commit 38aeb1d

Please sign in to comment.