WIP: duffle relocate of thick bundles

Rough first cut - currently fails some unit tests.

Fixes cnabio#705

Gopkg.toml

@@ -57,7 +57,7 @@
 
 [[constraint]]
   name = "github.com/pivotal/image-relocation"
-  revision = "532dd0b42e7a50010d7868364309cd314a2bb376"
+  revision = "3c9c32bb3d97fc213476f1c9846a99ca35ecba5d"
 
 [[override]]
   name = "github.com/google/go-containerregistry"

cmd/duffle/export.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"github.com/deislabs/duffle/pkg/imagestore/builder"
 	"io"
 
 	"github.com/deislabs/duffle/pkg/duffle/home"
@@ -76,20 +77,20 @@ func (ex *exportCmd) run() error {
 }
 
 func (ex *exportCmd) Export(bundlefile string, l loader.BundleLoader) error {
-	is, err := packager.NewImageStore(ex.thin)
+	bldr, err := builder.NewBuilder(ex.thin)
 	if err != nil {
 		return err
 	}
 
-	exp, err := packager.NewExporter(bundlefile, ex.dest, ex.home.Logs(), l, is)
+	exp, err := packager.NewExporter(bundlefile, ex.dest, ex.home.Logs(), l, bldr)
 	if err != nil {
 		return fmt.Errorf("Unable to set up exporter: %s", err)
 	}
 	if err := exp.Export(); err != nil {
 		return err
 	}
 	if ex.verbose {
-		fmt.Fprintf(ex.out, "Export logs: %s\n", exp.Logs)
+		fmt.Fprintf(ex.out, "Export logs: %s\n", exp.Logs())
 	}
 	return nil
 }

cmd/duffle/relocate.go

@@ -3,14 +3,20 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/deislabs/duffle/pkg/imagestore"
+	"github.com/deislabs/duffle/pkg/imagestore/builder"
+	"github.com/deislabs/duffle/pkg/loader"
+	"github.com/deislabs/duffle/pkg/packager"
+	"github.com/deislabs/duffle/pkg/relocator"
 	"io"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/pivotal/image-relocation/pkg/image"
 	"github.com/pivotal/image-relocation/pkg/pathmapping"
-	"github.com/pivotal/image-relocation/pkg/registry"
 
 	"github.com/deislabs/cnab-go/bundle"
 
@@ -51,8 +57,9 @@ type relocateCmd struct {
 	out  io.Writer
 
 	// dependencies
-	mapping        pathmapping.PathMapping
-	registryClient registry.Client
+	mapping           pathmapping.PathMapping
+	imageStoreBuilder imagestore.Builder
+	imageStore        imagestore.Store
 }
 
 func newRelocateCmd(w io.Writer) *cobra.Command {
@@ -81,7 +88,7 @@ duffle relocate path/to/bundle.json --relocation-mapping path/to/relmap.json --r
 			relocate.home = home.Home(homePath())
 
 			relocate.mapping = pathmapping.FlattenRepoPathPreserveTagDigest
-			relocate.registryClient = registry.NewRegistryClient()
+			relocate.imageStoreBuilder = builder.NewLocatingBuilder()
 
 			return relocate.run()
 		},
@@ -98,96 +105,78 @@ duffle relocate path/to/bundle.json --relocation-mapping path/to/relmap.json --r
 }
 
 func (r *relocateCmd) run() error {
-	bun, err := r.setup()
+	relMap := make(map[string]string)
+
+	rel, _, cleanup, err := r.setup()
 	if err != nil {
 		return err
 	}
+	defer cleanup()
 
-	if err := r.relocate(bun); err != nil {
+	if err := rel.Relocate(relMap); err != nil {
 		return err
 	}
 
-	return nil
+	return r.writeRelocationMapping(relMap)
 }
 
-func (r *relocateCmd) relocate(bun *bundle.Bundle) error {
-	relMap := make(map[string]string)
-	for i := range bun.InvocationImages {
-		ii := bun.InvocationImages[i]
-		modified, err := r.relocateImage(&ii.BaseImage, relMap)
+// The caller is responsible for running the returned cleanup function, which may delete the returned bundle.
+func (r *relocateCmd) setup() (*relocator.Relocator, *bundle.Bundle, func(), error) {
+	nop := func() {}
+	dest := ""
+	bundleFile, err := resolveBundleFilePath(r.inputBundle, r.home.String(), r.bundleIsFile)
+	if err != nil {
+		return nil, nil, nop, err
+	}
+
+	var bun *bundle.Bundle
+
+	if strings.HasSuffix(bundleFile, ".tgz") {
+		source, err := filepath.Abs(bundleFile)
 		if err != nil {
-			return err
+			return nil, nil, nop, err
 		}
-		if modified {
-			bun.InvocationImages[i] = ii
+
+		dest, err = ioutil.TempDir("", "duffle-relocate-unzip")
+		if err != nil {
+			return nil, nil, nop, err
 		}
-	}
 
-	for k := range bun.Images {
-		im := bun.Images[k]
-		modified, err := r.relocateImage(&im.BaseImage, relMap)
+		l := loader.NewLoader()
+		imp, err := packager.NewImporter(source, dest, l, false)
 		if err != nil {
-			return err
+			return nil, nil, nop, err
 		}
-		if modified {
-			bun.Images[k] = im
+		dest, bun, err = imp.Unzip()
+		if err != nil {
+			return nil, nil, nop, err
+		}
+	} else {
+		bun, err = loadBundle(bundleFile)
+		if err != nil {
+			return nil, nil, nop, err
 		}
 	}
 
-	return r.writeRelocationMapping(relMap)
-}
-
-func (r *relocateCmd) relocateImage(i *bundle.BaseImage, relMap map[string]string) (bool, error) {
-	if !isOCI(i.ImageType) && !isDocker(i.ImageType) {
-		return false, nil
-	}
-	// map the image name
-	n, err := image.NewName(i.Image)
-	if err != nil {
-		return false, err
+	if err = bun.Validate(); err != nil {
+		return nil, nil, nop, err
 	}
-	rn := r.mapping(r.repoPrefix, n)
 
-	// tag/push the image to its new repository
-	dig, err := r.registryClient.Copy(n, rn)
+	r.imageStore, err = r.imageStoreBuilder.ArchiveDir(dest).Build()
 	if err != nil {
-		return false, err
-	}
-	if i.Digest != "" && dig.String() != i.Digest {
-		// should not happen
-		return false, fmt.Errorf("digest of image %s not preserved: old digest %s; new digest %s", i.Image, i.Digest, dig.String())
+		return nil, nil, nop, err
 	}
 
-	// update the relocation map
-	relMap[i.Image] = rn.String()
-
-	return true, nil
-}
-
-func isOCI(imageType string) bool {
-	return imageType == "" || imageType == "oci"
-}
-
-func isDocker(imageType string) bool {
-	return imageType == "docker"
-}
-
-func (r *relocateCmd) setup() (*bundle.Bundle, error) {
-	bundleFile, err := resolveBundleFilePath(r.inputBundle, r.home.String(), r.bundleIsFile)
-	if err != nil {
-		return nil, err
+	mapping := func(i image.Name) image.Name {
+		return pathmapping.FlattenRepoPathPreserveTagDigest(r.repoPrefix, i)
 	}
 
-	bun, err := loadBundle(bundleFile)
+	reloc, err := relocator.NewRelocator(bun, mapping, r.imageStore)
 	if err != nil {
-		return nil, err
-	}
-
-	if err = bun.Validate(); err != nil {
-		return nil, err
+		return nil, nil, nop, err
 	}
 
-	return bun, nil
+	return reloc, bun, func() { os.RemoveAll(dest) }, nil
 }
 
 func (r *relocateCmd) writeRelocationMapping(relMap map[string]string) error {

pkg/imagestore/builder.go

@@ -0,0 +1,17 @@
+package imagestore
+
+import (
+	"io"
+)
+
+// Builder is a means of creating image stores.
+type Builder interface {
+	// ArchiveDir creates a fresh Builder with the given archive directory.
+	ArchiveDir(string) Builder
+
+	// Logs creates a fresh builder with the given log stream.
+	Logs(io.Writer) Builder
+
+	// Build creates an image store.
+	Build() (Store, error)
+}

pkg/imagestore/builder/builder.go

@@ -0,0 +1,69 @@
+package builder
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/deislabs/duffle/pkg/imagestore"
+	"github.com/deislabs/duffle/pkg/imagestore/ocilayout"
+	"github.com/deislabs/duffle/pkg/imagestore/remote"
+)
+
+// NewBuilder creates an image store builder which will, if necessary, create archive contents.
+func NewBuilder(remoteRepos bool) (imagestore.Builder, error) {
+	// infer the concrete type of the image store from the input parameters
+	if remoteRepos {
+		return remote.NewRemoteBuilder(), nil
+	}
+	return ocilayout.NewOciLayout(), nil
+}
+
+// NewLocatingBuilder creates an image store builder which will, if necessary, find existing archive contents.
+func NewLocatingBuilder() imagestore.Builder {
+	return &locatingBuilder{
+		archiveDir: "",
+		logs:       ioutil.Discard,
+	}
+}
+
+type locatingBuilder struct {
+	archiveDir string
+	logs       io.Writer
+}
+
+func (b *locatingBuilder) ArchiveDir(archiveDir string) imagestore.Builder {
+	return &locatingBuilder{
+		archiveDir: archiveDir,
+		logs:       b.logs,
+	}
+}
+
+func (b *locatingBuilder) Logs(logs io.Writer) imagestore.Builder {
+	return &locatingBuilder{
+		archiveDir: b.archiveDir,
+		logs:       logs,
+	}
+}
+
+func (b *locatingBuilder) Build() (imagestore.Store, error) {
+	if thin(b.archiveDir) {
+		return remote.NewRemoteBuilder().Build()
+	}
+
+	return ocilayout.LocateOciLayout(b.archiveDir)
+}
+
+func thin(archiveDir string) bool {
+	// If there is no archive directory, the bundle is thin
+	if archiveDir == "" {
+		return true
+	}
+
+	// If there is an archive directory, the bundle is thin if and only if the archive directory has no artifacts/
+	// subdirectory
+	layoutDir := filepath.Join(archiveDir, "artifacts")
+	_, err := os.Stat(layoutDir)
+	return os.IsNotExist(err)
+}

pkg/imagestore/imagestoremocks/builder.go

@@ -0,0 +1,27 @@
+package imagestoremocks
+
+import (
+	"io"
+
+	"github.com/deislabs/duffle/pkg/imagestore"
+)
+
+type MockBuilder struct {
+	ArchiveDirStub func(string)
+	LogsStub       func(io.Writer)
+	BuildStub      func() (imagestore.Store, error)
+}
+
+func (b *MockBuilder) ArchiveDir(archiveDir string) imagestore.Builder {
+	b.ArchiveDirStub(archiveDir)
+	return b
+}
+
+func (b *MockBuilder) Logs(logs io.Writer) imagestore.Builder {
+	b.LogsStub(logs)
+	return b
+}
+
+func (b *MockBuilder) Build() (imagestore.Store, error) {
+	return b.BuildStub()
+}

pkg/imagestore/imagestoremocks/store.go

@@ -0,0 +1,16 @@
+package imagestoremocks
+
+import "github.com/pivotal/image-relocation/pkg/image"
+
+type MockStore struct {
+	AddStub  func(im string) (string, error)
+	PushStub func(image.Digest, image.Name, image.Name) error
+}
+
+func (i *MockStore) Add(im string) (string, error) {
+	return i.AddStub(im)
+}
+
+func (i *MockStore) Push(dig image.Digest, src image.Name, dst image.Name) error {
+	return i.PushStub(dig, src, dst)
+}