From fbae6d93444cf424f8d0c3f2d8ff4af49e3ec8fd Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 16 Jun 2016 22:50:44 -0700 Subject: [PATCH 01/14] image/cas: Add a generic CAS interface And implement that interface for tarballs based on the specs image-layout. I plan on adding other backends later, but this is enough for a proof of concept. Also add a new oci-cas command so folks can access the new read functionality from the command line. In a subsequent commit, I'll replace the image/walker.go functionality with this new API. The Context interface follows the pattern recommended in [1], allowing callers to cancel long running actions (e.g. push/pull over the network for engine implementations that communicate with a remote store). blobPath's separator argument will allow us to use string(os.PathSeparator)) once we add directory support. [1]: https://blog.golang.org/context Signed-off-by: W. Trevor King --- .gitignore | 3 +- Makefile | 1 + cmd/oci-cas/get.go | 93 +++++++++++++++++++++++++++++++++++ cmd/oci-cas/main.go | 37 ++++++++++++++ image/cas/interface.go | 44 +++++++++++++++++ image/cas/layout/interface.go | 25 ++++++++++ image/cas/layout/main.go | 51 +++++++++++++++++++ image/cas/layout/tar.go | 90 +++++++++++++++++++++++++++++++++ 8 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 cmd/oci-cas/get.go create mode 100644 cmd/oci-cas/main.go create mode 100644 image/cas/interface.go create mode 100644 image/cas/layout/interface.go create mode 100644 image/cas/layout/main.go create mode 100644 image/cas/layout/tar.go diff --git a/.gitignore b/.gitignore index d897571..5dbca12 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/oci-cas /oci-create-runtime-bundle -/oci-unpack /oci-image-validate +/oci-unpack diff --git a/Makefile b/Makefile index f0f35f7..3cb46a9 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true) EPOCH_TEST_COMMIT ?= v0.2.0 TOOLS := \ + oci-cas \ oci-create-runtime-bundle \ oci-image-validate \ oci-unpack diff --git a/cmd/oci-cas/get.go b/cmd/oci-cas/get.go new file mode 100644 index 0000000..a779599 --- /dev/null +++ b/cmd/oci-cas/get.go @@ -0,0 +1,93 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type getCmd struct { + path string + digest string +} + +func newGetCmd() *cobra.Command { + state := &getCmd{} + + return &cobra.Command{ + Use: "get PATH DIGEST", + Short: "Retrieve a blob from the store", + Long: "Retrieve a blob from the store and write it to stdout.", + Run: state.Run, + } +} + +func (state *getCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and DIGEST must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.digest = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *getCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(state.path) + if err != nil { + return err + } + defer engine.Close() + + reader, err := engine.Get(ctx, state.digest) + if err != nil { + return err + } + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + + n, err := os.Stdout.Write(bytes) + if err != nil { + return err + } + if n < len(bytes) { + return fmt.Errorf("wrote %d of %d bytes", n, len(bytes)) + } + + return nil +} diff --git a/cmd/oci-cas/main.go b/cmd/oci-cas/main.go new file mode 100644 index 0000000..6a604e3 --- /dev/null +++ b/cmd/oci-cas/main.go @@ -0,0 +1,37 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{ + Use: "oci-cas", + Short: "Content-addressable storage manipulation", + } + + cmd.AddCommand(newGetCmd()) + + err := cmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/image/cas/interface.go b/image/cas/interface.go new file mode 100644 index 0000000..156ff48 --- /dev/null +++ b/image/cas/interface.go @@ -0,0 +1,44 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cas implements generic content-addressable storage. +package cas + +import ( + "io" + + "golang.org/x/net/context" +) + +// Engine represents a content-addressable storage engine. +type Engine interface { + + // Put adds a new blob to the store. The action is idempotent; a + // nil return means "that content is stored at DIGEST" without + // implying "because of your Put()". + Put(ctx context.Context, reader io.Reader) (digest string, err error) + + // Get returns a reader for retrieving a blob from the store. + // Returns os.ErrNotExist if the digest is not found. + Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) + + // Delete removes a blob from the store. The action is idempotent; a + // nil return means "that content is not in the store" without + // implying "because of your Delete()". + Delete(ctx context.Context, digest string) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close() (err error) +} diff --git a/image/cas/layout/interface.go b/image/cas/layout/interface.go new file mode 100644 index 0000000..dda72d6 --- /dev/null +++ b/image/cas/layout/interface.go @@ -0,0 +1,25 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" +) + +// ReadWriteSeekCloser wraps the Read, Write, Seek, and Close methods. +type ReadWriteSeekCloser interface { + io.ReadWriteSeeker + io.Closer +} diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go new file mode 100644 index 0000000..0a890a4 --- /dev/null +++ b/image/cas/layout/main.go @@ -0,0 +1,51 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout implements the cas interface using the image-spec's +// image-layout [1]. +// +// [1]: /~https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout + +import ( + "fmt" + "os" + "strings" + + "github.com/opencontainers/image-tools/image/cas" +) + +// NewEngine instantiates an engine with the appropriate backend (tar, +// HTTP, ...). +func NewEngine(path string) (engine cas.Engine, err error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + return NewTarEngine(file) +} + +// blobPath returns the PATH to the DIGEST blob. SEPARATOR selects +// the path separator used between components. +func blobPath(digest string, separator string) (path string, err error) { + fields := strings.SplitN(digest, ":", 2) + if len(fields) != 2 { + return "", fmt.Errorf("invalid digest: %q, %v", digest, fields) + } + algorithm := fields[0] + hash := fields[1] + components := []string{".", "blobs", algorithm, hash} + return strings.Join(components, separator), nil +} diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go new file mode 100644 index 0000000..68fccac --- /dev/null +++ b/image/cas/layout/tar.go @@ -0,0 +1,90 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "errors" + "io" + "io/ioutil" + "os" + + "github.com/opencontainers/image-tools/image/cas" + "golang.org/x/net/context" +) + +// TarEngine is a cas.Engine backed by a tar file. +type TarEngine struct { + file ReadWriteSeekCloser +} + +// NewTarEngine returns a new TarEngine. +func NewTarEngine(file ReadWriteSeekCloser) (engine cas.Engine, err error) { + engine = &TarEngine{ + file: file, + } + + return engine, nil +} + +// Put adds a new blob to the store. +func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { + // FIXME + return "", errors.New("TarEngine.Put is not supported yet") +} + +// Get returns a reader for retrieving a blob from the store. +func (engine *TarEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) { + targetName, err := blobPath(digest, "/") + if err != nil { + return nil, err + } + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil, err + } + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return nil, os.ErrNotExist + } else if err != nil { + return nil, err + } + + if header.Name == targetName { + return ioutil.NopCloser(tarReader), nil + } + } +} + +// Delete removes a blob from the store. +func (engine *TarEngine) Delete(ctx context.Context, digest string) (err error) { + // FIXME + return errors.New("TarEngine.Delete is not supported yet") +} + +// Close releases resources held by the engine. +func (engine *TarEngine) Close() (err error) { + return engine.file.Close() +} From 1a2c78f88726148a31b34fb1912acbd5b8d6a9f5 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 00:38:03 -0700 Subject: [PATCH 02/14] image/refs: Add a generic name-based reference interface And implement that interface for tarballs based on the specs image-layout. I plan on adding other backends later, but this is enough for a proof of concept. Also add a new oci-refs command so folks can access the new read functionality from the command line. The Engine.List interface uses a callback instead of returning channels or a slice. Benefits vs. returning a slice of names: * There's no need to allocate a slice for the results, so calls with large (or negative) 'size' values can be made without consuming large amounts of memory. * The name collection and processing can happen concurrently, so: * We don't waste cycles collecting names we won't use. * Slow collection can happen in the background if/when the consumer is blocked on something else. The benefit of using callbacks vs. returning name and error channels (as discussed in [1]) is more of a trade-off. Stephen Day [2] and JT Olds [3] don't like channel's internal locks. Dave Cheney doesn't have a problem with them [4]. Which approach is more efficient for a given situation depends on how expensive it is for the engine to find the next key and how expensive it is to act on a returned name. If both are expensive, you want goroutines in there somewhere to get concurrent execution, and channels will help those goroutines communicate. When either action is fast (or both are fast), channels are unnecessary overhead. By using a callback in the interface, we avoid baking in the overhead. Folks who want concurrent execution can initialize their own channel, launch List in a goroutine, and use the callback to inject names into their channel. In a subsequent commit, I'll replace the image/walker.go functionality with this new API. I'd prefer casLayout for the imported package, but Stephen doesn't want camelCase for package names [5]. [1]: https://blog.golang.org/pipelines [2]: /~https://github.com/opencontainers/image-spec/pull/159#discussion_r76874690 [3]: http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/ [4]: https://groups.google.com/d/msg/golang-nuts/LM648yrPpck/idyupwodAwAJ Subject: Re: [go-nuts] Re: "Go channels are bad and you should feel bad" Date: Wed, 2 Mar 2016 16:04:13 -0800 (PST) Message-Id: [5]: /~https://github.com/opencontainers/image-spec/pull/159#discussion_r76720225 Signed-off-by: W. Trevor King --- .gitignore | 1 + Makefile | 1 + cmd/oci-refs/get.go | 78 +++++++++++++++++++++ cmd/oci-refs/list.go | 78 +++++++++++++++++++++ cmd/oci-refs/main.go | 38 ++++++++++ image/refs/interface.go | 79 +++++++++++++++++++++ image/refs/layout/main.go | 44 ++++++++++++ image/refs/layout/tar.go | 144 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 463 insertions(+) create mode 100644 cmd/oci-refs/get.go create mode 100644 cmd/oci-refs/list.go create mode 100644 cmd/oci-refs/main.go create mode 100644 image/refs/interface.go create mode 100644 image/refs/layout/main.go create mode 100644 image/refs/layout/tar.go diff --git a/.gitignore b/.gitignore index 5dbca12..fe4e8e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /oci-cas /oci-create-runtime-bundle /oci-image-validate +/oci-refs /oci-unpack diff --git a/Makefile b/Makefile index 3cb46a9..ed26eae 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ TOOLS := \ oci-cas \ oci-create-runtime-bundle \ oci-image-validate \ + oci-refs \ oci-unpack default: help diff --git a/cmd/oci-refs/get.go b/cmd/oci-refs/get.go new file mode 100644 index 0000000..0994ef1 --- /dev/null +++ b/cmd/oci-refs/get.go @@ -0,0 +1,78 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type getCmd struct { + path string + name string +} + +func newGetCmd() *cobra.Command { + state := &getCmd{} + + return &cobra.Command{ + Use: "get PATH NAME", + Short: "Retrieve a reference from the store", + Run: state.Run, + } +} + +func (state *getCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and NAME must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *getCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(state.path) + if err != nil { + return err + } + defer engine.Close() + + descriptor, err := engine.Get(ctx, state.name) + if err != nil { + return err + } + + return json.NewEncoder(os.Stdout).Encode(&descriptor) +} diff --git a/cmd/oci-refs/list.go b/cmd/oci-refs/list.go new file mode 100644 index 0000000..0e25e82 --- /dev/null +++ b/cmd/oci-refs/list.go @@ -0,0 +1,78 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type listCmd struct { + path string +} + +func newListCmd() *cobra.Command { + state := &listCmd{} + + return &cobra.Command{ + Use: "list PATH", + Short: "Return available names from the store.", + Run: state.Run, + } +} + +func (state *listCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + fmt.Fprintln(os.Stderr, "PATH must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *listCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.List(ctx, "", -1, 0, state.printName) +} + +func (state *listCmd) printName(ctx context.Context, name string) (err error) { + n, err := fmt.Fprintln(os.Stdout, name) + if n < len(name) { + return fmt.Errorf("wrote %d of %d name", n, len(name)) + } + return err +} diff --git a/cmd/oci-refs/main.go b/cmd/oci-refs/main.go new file mode 100644 index 0000000..cb2cc7d --- /dev/null +++ b/cmd/oci-refs/main.go @@ -0,0 +1,38 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{ + Use: "oci-refs", + Short: "Name-based reference manipulation", + } + + cmd.AddCommand(newGetCmd()) + cmd.AddCommand(newListCmd()) + + err := cmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/image/refs/interface.go b/image/refs/interface.go new file mode 100644 index 0000000..92472eb --- /dev/null +++ b/image/refs/interface.go @@ -0,0 +1,79 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package refs implements generic name-based reference access. +package refs + +import ( + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// ListNameCallback templates an Engine.List callback used for +// processing names. See Engine.List for more details. +type ListNameCallback func(ctx context.Context, name string) (err error) + +// Engine represents a name-based reference storage engine. +type Engine interface { + + // Put adds a new reference to the store. The action is idempotent; + // a nil return means "that descriptor is stored at NAME" without + // implying "because of your Put()". + Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) + + // Get returns a reference from the store. Returns os.ErrNotExist + // if the name is not found. + Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) + + // List returns available names from the store. + // + // Results are sorted alphabetically. + // + // Arguments: + // + // * ctx: gives callers a way to gracefully cancel a long-running + // list. + // * prefix: limits the result set to names starting with that + // value. + // * size: limits the length of the result set to the first 'size' + // matches. A value of -1 means "all results". + // * from: shifts the result set to start from the 'from'th match. + // * nameCallback: called for every matching name. List returns any + // errors returned by nameCallback and aborts further listing. + // + // For example, a store with names like: + // + // * 123 + // * abcd + // * abce + // * abcf + // * abcg + // + // will have the following call/result pairs: + // + // * List(ctx, "", -1, 0, printName) -> "123", "abcd", "abce", "abcf", "abcg" + // * List(ctx, "", 2, 0, printName) -> "123", "abcd" + // * List(ctx, "", 2, 1, printName) -> "abcd", "abce" + // * List(ctx,"abc", 2, 1, printName) -> "abce", "abcf" + List(ctx context.Context, prefix string, size int, from int, nameCallback ListNameCallback) (err error) + + // Delete removes a reference from the store. The action is + // idempotent; a nil return means "that reference is not in the + // store" without implying "because of your Delete()". + Delete(ctx context.Context, name string) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close() (err error) +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go new file mode 100644 index 0000000..4117690 --- /dev/null +++ b/image/refs/layout/main.go @@ -0,0 +1,44 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout implements the refs interface using the image-spec's +// image-layout [1]. +// +// [1]: /~https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout + +import ( + "os" + "strings" + + "github.com/opencontainers/image-tools/image/refs" +) + +// NewEngine instantiates an engine with the appropriate backend (tar, +// HTTP, ...). +func NewEngine(path string) (engine refs.Engine, err error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + return NewTarEngine(file) +} + +// refPath returns the PATH to the NAME reference. SEPARATOR selects +// the path separator used between components. +func refPath(name string, separator string) (path string, err error) { + components := []string{".", "refs", name} + return strings.Join(components, separator), nil +} diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go new file mode 100644 index 0000000..6719a77 --- /dev/null +++ b/image/refs/layout/tar.go @@ -0,0 +1,144 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "encoding/json" + "errors" + "io" + "os" + "strings" + + "github.com/opencontainers/image-spec/specs-go" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/opencontainers/image-tools/image/refs" + "golang.org/x/net/context" +) + +// TarEngine is a refs.Engine backed by a tar file. +type TarEngine struct { + file caslayout.ReadWriteSeekCloser +} + +// NewTarEngine returns a new TarEngine. +func NewTarEngine(file caslayout.ReadWriteSeekCloser) (engine refs.Engine, err error) { + engine = &TarEngine{ + file: file, + } + + return engine, nil +} + +// Put adds a new reference to the store. +func (engine *TarEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { + // FIXME + return errors.New("TarEngine.Put is not supported yet") +} + +// Get returns a reference from the store. +func (engine *TarEngine) Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) { + targetName, err := refPath(name, "/") + if err != nil { + return nil, err + } + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil, err + } + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return nil, os.ErrNotExist + } else if err != nil { + return nil, err + } + + if header.Name == targetName { + decoder := json.NewDecoder(tarReader) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err + } + return &desc, nil + } + } +} + +// List returns available names from the store. +func (engine *TarEngine) List(ctx context.Context, prefix string, size int, from int, nameCallback refs.ListNameCallback) (err error) { + var i = 0 + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil + } + + pathPrefix, err := refPath(prefix, "/") + if err != nil { + return nil + } + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var header *tar.Header + header, err = tarReader.Next() + if err == io.EOF { + return nil + } else if err != nil { + return err + } + + if strings.HasPrefix(header.Name, pathPrefix) && len(header.Name) > 7 { + i++ + if i > from { + err = nameCallback(ctx, header.Name[7:]) + if err != nil { + return err + } + if i-from == size { + return nil + } + } + } + } +} + +// Delete removes a reference from the store. +func (engine *TarEngine) Delete(ctx context.Context, name string) (err error) { + // FIXME + return errors.New("TarEngine.Delete is not supported yet") +} + +// Close releases resources held by the engine. +func (engine *TarEngine) Close() (err error) { + return engine.file.Close() +} From b27e428e988e4aeb793c154fe9d6d93deaa8bcca Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 10:43:20 -0700 Subject: [PATCH 03/14] specs-go: Add ImageLayoutVersion and check oci-layout in tar engines Collect the shared stuff in the image/layout utility package. Signed-off-by: W. Trevor King --- cmd/oci-cas/get.go | 2 +- cmd/oci-refs/get.go | 2 +- cmd/oci-refs/list.go | 2 +- image/cas/layout/main.go | 5 +-- image/cas/layout/tar.go | 10 ++++-- image/layout/layout.go | 22 +++++++++++++ image/layout/tar.go | 66 +++++++++++++++++++++++++++++++++++++++ image/refs/layout/main.go | 5 +-- image/refs/layout/tar.go | 10 ++++-- 9 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 image/layout/layout.go create mode 100644 image/layout/tar.go diff --git a/cmd/oci-cas/get.go b/cmd/oci-cas/get.go index a779599..3a7d139 100644 --- a/cmd/oci-cas/get.go +++ b/cmd/oci-cas/get.go @@ -64,7 +64,7 @@ func (state *getCmd) Run(cmd *cobra.Command, args []string) { func (state *getCmd) run() (err error) { ctx := context.Background() - engine, err := layout.NewEngine(state.path) + engine, err := layout.NewEngine(ctx, state.path) if err != nil { return err } diff --git a/cmd/oci-refs/get.go b/cmd/oci-refs/get.go index 0994ef1..5daf671 100644 --- a/cmd/oci-refs/get.go +++ b/cmd/oci-refs/get.go @@ -63,7 +63,7 @@ func (state *getCmd) Run(cmd *cobra.Command, args []string) { func (state *getCmd) run() (err error) { ctx := context.Background() - engine, err := layout.NewEngine(state.path) + engine, err := layout.NewEngine(ctx, state.path) if err != nil { return err } diff --git a/cmd/oci-refs/list.go b/cmd/oci-refs/list.go index 0e25e82..4e5e6d3 100644 --- a/cmd/oci-refs/list.go +++ b/cmd/oci-refs/list.go @@ -60,7 +60,7 @@ func (state *listCmd) Run(cmd *cobra.Command, args []string) { func (state *listCmd) run() (err error) { ctx := context.Background() - engine, err := layout.NewEngine(state.path) + engine, err := layout.NewEngine(ctx, state.path) if err != nil { return err } diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go index 0a890a4..1ab7b04 100644 --- a/image/cas/layout/main.go +++ b/image/cas/layout/main.go @@ -24,17 +24,18 @@ import ( "strings" "github.com/opencontainers/image-tools/image/cas" + "golang.org/x/net/context" ) // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). -func NewEngine(path string) (engine cas.Engine, err error) { +func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) { file, err := os.Open(path) if err != nil { return nil, err } - return NewTarEngine(file) + return NewTarEngine(ctx, file) } // blobPath returns the PATH to the DIGEST blob. SEPARATOR selects diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go index 68fccac..36aed74 100644 --- a/image/cas/layout/tar.go +++ b/image/cas/layout/tar.go @@ -22,6 +22,7 @@ import ( "os" "github.com/opencontainers/image-tools/image/cas" + "github.com/opencontainers/image-tools/image/layout" "golang.org/x/net/context" ) @@ -31,11 +32,16 @@ type TarEngine struct { } // NewTarEngine returns a new TarEngine. -func NewTarEngine(file ReadWriteSeekCloser) (engine cas.Engine, err error) { - engine = &TarEngine{ +func NewTarEngine(ctx context.Context, file ReadWriteSeekCloser) (eng cas.Engine, err error) { + engine := &TarEngine{ file: file, } + err = layout.CheckTarVersion(ctx, engine.file) + if err != nil { + return nil, err + } + return engine, nil } diff --git a/image/layout/layout.go b/image/layout/layout.go new file mode 100644 index 0000000..e6989fe --- /dev/null +++ b/image/layout/layout.go @@ -0,0 +1,22 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout defines utility code shared by refs/layout and cas/layout. +package layout + +// ImageLayoutVersion represents the oci-version content for the image +// layout format. +type ImageLayoutVersion struct { + Version string `json:"imageLayoutVersion"` +} diff --git a/image/layout/tar.go b/image/layout/tar.go new file mode 100644 index 0000000..46ab8b3 --- /dev/null +++ b/image/layout/tar.go @@ -0,0 +1,66 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "golang.org/x/net/context" +) + +// CheckTarVersion walks a tarball pointed to by reader and returns an +// error if oci-layout is missing or has unrecognized content. +func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { + _, err = reader.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + + tarReader := tar.NewReader(reader) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + + if header.Name == "./oci-layout" { + decoder := json.NewDecoder(tarReader) + var version ImageLayoutVersion + err = decoder.Decode(&version) + if err != nil { + return err + } + if version.Version != "1.0.0" { + return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) + } + + return nil + } + } +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go index 4117690..81f885c 100644 --- a/image/refs/layout/main.go +++ b/image/refs/layout/main.go @@ -23,17 +23,18 @@ import ( "strings" "github.com/opencontainers/image-tools/image/refs" + "golang.org/x/net/context" ) // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). -func NewEngine(path string) (engine refs.Engine, err error) { +func NewEngine(ctx context.Context, path string) (engine refs.Engine, err error) { file, err := os.Open(path) if err != nil { return nil, err } - return NewTarEngine(file) + return NewTarEngine(ctx, file) } // refPath returns the PATH to the NAME reference. SEPARATOR selects diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go index 6719a77..daa3507 100644 --- a/image/refs/layout/tar.go +++ b/image/refs/layout/tar.go @@ -24,6 +24,7 @@ import ( "github.com/opencontainers/image-spec/specs-go" caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" "github.com/opencontainers/image-tools/image/refs" "golang.org/x/net/context" ) @@ -34,11 +35,16 @@ type TarEngine struct { } // NewTarEngine returns a new TarEngine. -func NewTarEngine(file caslayout.ReadWriteSeekCloser) (engine refs.Engine, err error) { - engine = &TarEngine{ +func NewTarEngine(ctx context.Context, file caslayout.ReadWriteSeekCloser) (eng refs.Engine, err error) { + engine := &TarEngine{ file: file, } + err = imagelayout.CheckTarVersion(ctx, engine.file) + if err != nil { + return nil, err + } + return engine, nil } From 9263d6ef8ed40cc2bf69971b031da58a41dc68a9 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 9 Jul 2016 21:34:44 -0700 Subject: [PATCH 04/14] image/layout/tar.go: Add TarEntryByName Making the CAS/refs Get implementations more DRY. Signed-off-by: W. Trevor King --- image/cas/layout/tar.go | 24 +++--------------- image/layout/tar.go | 55 +++++++++++++++++++++++++--------------- image/refs/layout/tar.go | 32 +++++------------------ 3 files changed, 45 insertions(+), 66 deletions(-) diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go index 36aed74..f4eab92 100644 --- a/image/cas/layout/tar.go +++ b/image/cas/layout/tar.go @@ -15,11 +15,11 @@ package layout import ( - "archive/tar" "errors" "io" "io/ioutil" "os" + "strings" "github.com/opencontainers/image-tools/image/cas" "github.com/opencontainers/image-tools/image/layout" @@ -58,30 +58,12 @@ func (engine *TarEngine) Get(ctx context.Context, digest string) (reader io.Read return nil, err } - _, err = engine.file.Seek(0, os.SEEK_SET) + _, tarReader, err := layout.TarEntryByName(ctx, engine.file, targetName) if err != nil { return nil, err } - tarReader := tar.NewReader(engine.file) - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - header, err := tarReader.Next() - if err == io.EOF { - return nil, os.ErrNotExist - } else if err != nil { - return nil, err - } - - if header.Name == targetName { - return ioutil.NopCloser(tarReader), nil - } - } + return ioutil.NopCloser(tarReader), nil } // Delete removes a blob from the store. diff --git a/image/layout/tar.go b/image/layout/tar.go index 46ab8b3..270feba 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -25,42 +25,57 @@ import ( "golang.org/x/net/context" ) -// CheckTarVersion walks a tarball pointed to by reader and returns an -// error if oci-layout is missing or has unrecognized content. -func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { +// TarEntryByName walks a tarball pointed to by reader, finds an +// entry matching the given name, and returns the header and reader +// for that entry. Returns os.ErrNotExist if the path is not found. +func TarEntryByName(ctx context.Context, reader io.ReadSeeker, name string) (header *tar.Header, tarReader *tar.Reader, err error) { _, err = reader.Seek(0, os.SEEK_SET) if err != nil { - return err + return nil, nil, err } - tarReader := tar.NewReader(reader) + tarReader = tar.NewReader(reader) for { select { case <-ctx.Done(): - return ctx.Err() + return nil, nil, ctx.Err() default: } header, err := tarReader.Next() if err == io.EOF { - return errors.New("oci-layout not found") + return nil, nil, os.ErrNotExist } if err != nil { - return err + return nil, nil, err } - if header.Name == "./oci-layout" { - decoder := json.NewDecoder(tarReader) - var version ImageLayoutVersion - err = decoder.Decode(&version) - if err != nil { - return err - } - if version.Version != "1.0.0" { - return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) - } - - return nil + if header.Name == name { + return header, tarReader, nil } } } + +// CheckTarVersion walks a tarball pointed to by reader and returns an +// error if oci-layout is missing or has unrecognized content. +func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { + _, tarReader, err := TarEntryByName(ctx, reader, "./oci-layout") + if err == os.ErrNotExist { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + + decoder := json.NewDecoder(tarReader) + var version ImageLayoutVersion + err = decoder.Decode(&version) + if err != nil { + return err + } + if version.Version != "1.0.0" { + return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) + } + + return nil +} diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go index daa3507..12f903c 100644 --- a/image/refs/layout/tar.go +++ b/image/refs/layout/tar.go @@ -61,36 +61,18 @@ func (engine *TarEngine) Get(ctx context.Context, name string) (descriptor *spec return nil, err } - _, err = engine.file.Seek(0, os.SEEK_SET) + _, tarReader, err := imagelayout.TarEntryByName(ctx, engine.file, targetName) if err != nil { return nil, err } - tarReader := tar.NewReader(engine.file) - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - header, err := tarReader.Next() - if err == io.EOF { - return nil, os.ErrNotExist - } else if err != nil { - return nil, err - } - - if header.Name == targetName { - decoder := json.NewDecoder(tarReader) - var desc specs.Descriptor - err = decoder.Decode(&desc) - if err != nil { - return nil, err - } - return &desc, nil - } + decoder := json.NewDecoder(tarReader) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err } + return &desc, nil } // List returns available names from the store. From fe8f7cddcd650c7cc03f1d64a5ad57a3a90cf5bb Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 26 Jul 2016 21:25:12 -0700 Subject: [PATCH 05/14] image/cas/put: Add a PutJSON helper A fair amount of image setup involves pushing JSON objects to CAS, so provide a convenient wrapper around that. This implementation could be improved, with at least: * Consistent key sorts, etc. to increase the chances of matching an existing CAS blob. * Streaming the marshaled JSON into the engine to avoid serializing it in memory before passing it into Engine.Put. But the API is fine, and we can improve the implementation as we go. Signed-off-by: W. Trevor King --- image/cas/put.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 image/cas/put.go diff --git a/image/cas/put.go b/image/cas/put.go new file mode 100644 index 0000000..0f61881 --- /dev/null +++ b/image/cas/put.go @@ -0,0 +1,47 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cas + +import ( + "bytes" + "encoding/json" + + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// PutJSON writes a generic JSON object to content-addressable storage +// and returns a Descriptor referencing it. +func PutJSON(ctx context.Context, engine Engine, data interface{}, mediaType string) (descriptor *specs.Descriptor, err error) { + jsonBytes, err := json.Marshal(data) + if err != nil { + return nil, err + } + size := len(jsonBytes) + size64 := int64(size) // panics on overflow + + reader := bytes.NewReader(jsonBytes) + digest, err := engine.Put(ctx, reader) + if err != nil { + return nil, err + } + + descriptor = &specs.Descriptor{ + MediaType: mediaType, + Digest: digest, + Size: size64, + } + return descriptor, nil +} From 753408bc0015b0c3f5a6c33c7a498c35a283ad9e Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Jul 2016 15:28:44 -0700 Subject: [PATCH 06/14] vendor: Bundle golang.org/x/net/context Generated with: $ make install.tools $ make update-deps $ git checkout HEAD -- vendor/github.com/spf13/pflag $ git checkout HEAD -- vendor/github.com/runtime-spec I rolled back the other changes because I haven't checked for compatibility issues due to the upgrades. I also rolled back the hash bumps for those packages in glide.lock. Signed-off-by: W. Trevor King --- glide.lock | 6 +- vendor/golang.org/x/net/LICENSE | 27 ++ vendor/golang.org/x/net/PATENTS | 22 ++ vendor/golang.org/x/net/context/context.go | 156 ++++++++++ vendor/golang.org/x/net/context/go17.go | 72 +++++ vendor/golang.org/x/net/context/pre_go17.go | 300 ++++++++++++++++++++ 6 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 vendor/golang.org/x/net/LICENSE create mode 100644 vendor/golang.org/x/net/PATENTS create mode 100644 vendor/golang.org/x/net/context/context.go create mode 100644 vendor/golang.org/x/net/context/go17.go create mode 100644 vendor/golang.org/x/net/context/pre_go17.go diff --git a/glide.lock b/glide.lock index bffc373..3301bc4 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 14550151b754f92de80e864f26da93e8adfce71537c9cb7883b0bdd67e541453 -updated: 2016-09-15T10:25:30.038113538+02:00 +updated: 2016-09-16T00:10:45.625507345-07:00 imports: - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 @@ -29,4 +29,8 @@ imports: version: e7a2449258501866491620d4f47472e6100ca551 subpackages: - errorutil +- name: golang.org/x/net + version: 71a035914f99bb58fe82eac0f1289f10963d876c + subpackages: + - context testImports: [] diff --git a/vendor/golang.org/x/net/LICENSE b/vendor/golang.org/x/net/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/golang.org/x/net/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/net/PATENTS b/vendor/golang.org/x/net/PATENTS new file mode 100644 index 0000000..7330990 --- /dev/null +++ b/vendor/golang.org/x/net/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go new file mode 100644 index 0000000..134654c --- /dev/null +++ b/vendor/golang.org/x/net/context/context.go @@ -0,0 +1,156 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package context defines the Context type, which carries deadlines, +// cancelation signals, and other request-scoped values across API boundaries +// and between processes. +// +// Incoming requests to a server should create a Context, and outgoing calls to +// servers should accept a Context. The chain of function calls between must +// propagate the Context, optionally replacing it with a modified copy created +// using WithDeadline, WithTimeout, WithCancel, or WithValue. +// +// Programs that use Contexts should follow these rules to keep interfaces +// consistent across packages and enable static analysis tools to check context +// propagation: +// +// Do not store Contexts inside a struct type; instead, pass a Context +// explicitly to each function that needs it. The Context should be the first +// parameter, typically named ctx: +// +// func DoSomething(ctx context.Context, arg Arg) error { +// // ... use ctx ... +// } +// +// Do not pass a nil Context, even if a function permits it. Pass context.TODO +// if you are unsure about which Context to use. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +// +// The same Context may be passed to functions running in different goroutines; +// Contexts are safe for simultaneous use by multiple goroutines. +// +// See http://blog.golang.org/context for example code for a server that uses +// Contexts. +package context // import "golang.org/x/net/context" + +import "time" + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + // + // WithCancel arranges for Done to be closed when cancel is called; + // WithDeadline arranges for Done to be closed when the deadline + // expires; WithTimeout arranges for Done to be closed when the timeout + // elapses. + // + // Done is provided for use in select statements: + // + // // Stream generates values with DoSomething and sends them to out + // // until DoSomething returns an error or ctx.Done is closed. + // func Stream(ctx context.Context, out chan<- Value) error { + // for { + // v, err := DoSomething(ctx) + // if err != nil { + // return err + // } + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case out <- v: + // } + // } + // } + // + // See http://blog.golang.org/pipelines for more examples of how to use + // a Done channel for cancelation. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + // + // A key identifies a specific value in a Context. Functions that wish + // to store values in Context typically allocate a key in a global + // variable then use that key as the argument to context.WithValue and + // Context.Value. A key can be any type that supports equality; + // packages should define keys as an unexported type to avoid + // collisions. + // + // Packages that define a Context key should provide type-safe accessors + // for the values stores using that key: + // + // // Package user defines a User type that's stored in Contexts. + // package user + // + // import "golang.org/x/net/context" + // + // // User is the type of value stored in the Contexts. + // type User struct {...} + // + // // key is an unexported type for keys defined in this package. + // // This prevents collisions with keys defined in other packages. + // type key int + // + // // userKey is the key for user.User values in Contexts. It is + // // unexported; clients use user.NewContext and user.FromContext + // // instead of using this key directly. + // var userKey key = 0 + // + // // NewContext returns a new Context that carries value u. + // func NewContext(ctx context.Context, u *User) context.Context { + // return context.WithValue(ctx, userKey, u) + // } + // + // // FromContext returns the User value stored in ctx, if any. + // func FromContext(ctx context.Context) (*User, bool) { + // u, ok := ctx.Value(userKey).(*User) + // return u, ok + // } + Value(key interface{}) interface{} +} + +// Background returns a non-nil, empty Context. It is never canceled, has no +// values, and has no deadline. It is typically used by the main function, +// initialization, and tests, and as the top-level Context for incoming +// requests. +func Background() Context { + return background +} + +// TODO returns a non-nil, empty Context. Code should use context.TODO when +// it's unclear which Context to use or it is not yet available (because the +// surrounding function has not yet been extended to accept a Context +// parameter). TODO is recognized by static analysis tools that determine +// whether Contexts are propagated correctly in a program. +func TODO() Context { + return todo +} + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc func() diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go new file mode 100644 index 0000000..f8cda19 --- /dev/null +++ b/vendor/golang.org/x/net/context/go17.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package context + +import ( + "context" // standard library's context, as of Go 1.7 + "time" +) + +var ( + todo = context.TODO() + background = context.Background() +) + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = context.Canceled + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = context.DeadlineExceeded + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + ctx, f := context.WithCancel(parent) + return ctx, CancelFunc(f) +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + ctx, f := context.WithDeadline(parent, deadline) + return ctx, CancelFunc(f) +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return context.WithValue(parent, key, val) +} diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go new file mode 100644 index 0000000..5a30aca --- /dev/null +++ b/vendor/golang.org/x/net/context/pre_go17.go @@ -0,0 +1,300 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.7 + +package context + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// An emptyCtx is never canceled, has no values, and has no deadline. It is not +// struct{}, since vars of this type must have distinct addresses. +type emptyCtx int + +func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { + return +} + +func (*emptyCtx) Done() <-chan struct{} { + return nil +} + +func (*emptyCtx) Err() error { + return nil +} + +func (*emptyCtx) Value(key interface{}) interface{} { + return nil +} + +func (e *emptyCtx) String() string { + switch e { + case background: + return "context.Background" + case todo: + return "context.TODO" + } + return "unknown empty Context" +} + +var ( + background = new(emptyCtx) + todo = new(emptyCtx) +) + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = errors.New("context canceled") + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = errors.New("context deadline exceeded") + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + c := newCancelCtx(parent) + propagateCancel(parent, c) + return c, func() { c.cancel(true, Canceled) } +} + +// newCancelCtx returns an initialized cancelCtx. +func newCancelCtx(parent Context) *cancelCtx { + return &cancelCtx{ + Context: parent, + done: make(chan struct{}), + } +} + +// propagateCancel arranges for child to be canceled when parent is. +func propagateCancel(parent Context, child canceler) { + if parent.Done() == nil { + return // parent is never canceled + } + if p, ok := parentCancelCtx(parent); ok { + p.mu.Lock() + if p.err != nil { + // parent has already been canceled + child.cancel(false, p.err) + } else { + if p.children == nil { + p.children = make(map[canceler]bool) + } + p.children[child] = true + } + p.mu.Unlock() + } else { + go func() { + select { + case <-parent.Done(): + child.cancel(false, parent.Err()) + case <-child.Done(): + } + }() + } +} + +// parentCancelCtx follows a chain of parent references until it finds a +// *cancelCtx. This function understands how each of the concrete types in this +// package represents its parent. +func parentCancelCtx(parent Context) (*cancelCtx, bool) { + for { + switch c := parent.(type) { + case *cancelCtx: + return c, true + case *timerCtx: + return c.cancelCtx, true + case *valueCtx: + parent = c.Context + default: + return nil, false + } + } +} + +// removeChild removes a context from its parent. +func removeChild(parent Context, child canceler) { + p, ok := parentCancelCtx(parent) + if !ok { + return + } + p.mu.Lock() + if p.children != nil { + delete(p.children, child) + } + p.mu.Unlock() +} + +// A canceler is a context type that can be canceled directly. The +// implementations are *cancelCtx and *timerCtx. +type canceler interface { + cancel(removeFromParent bool, err error) + Done() <-chan struct{} +} + +// A cancelCtx can be canceled. When canceled, it also cancels any children +// that implement canceler. +type cancelCtx struct { + Context + + done chan struct{} // closed by the first cancel call. + + mu sync.Mutex + children map[canceler]bool // set to nil by the first cancel call + err error // set to non-nil by the first cancel call +} + +func (c *cancelCtx) Done() <-chan struct{} { + return c.done +} + +func (c *cancelCtx) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func (c *cancelCtx) String() string { + return fmt.Sprintf("%v.WithCancel", c.Context) +} + +// cancel closes c.done, cancels each of c's children, and, if +// removeFromParent is true, removes c from its parent's children. +func (c *cancelCtx) cancel(removeFromParent bool, err error) { + if err == nil { + panic("context: internal error: missing cancel error") + } + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return // already canceled + } + c.err = err + close(c.done) + for child := range c.children { + // NOTE: acquiring the child's lock while holding parent's lock. + child.cancel(false, err) + } + c.children = nil + c.mu.Unlock() + + if removeFromParent { + removeChild(c.Context, c) + } +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { + // The current deadline is already sooner than the new one. + return WithCancel(parent) + } + c := &timerCtx{ + cancelCtx: newCancelCtx(parent), + deadline: deadline, + } + propagateCancel(parent, c) + d := deadline.Sub(time.Now()) + if d <= 0 { + c.cancel(true, DeadlineExceeded) // deadline has already passed + return c, func() { c.cancel(true, Canceled) } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.err == nil { + c.timer = time.AfterFunc(d, func() { + c.cancel(true, DeadlineExceeded) + }) + } + return c, func() { c.cancel(true, Canceled) } +} + +// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to +// implement Done and Err. It implements cancel by stopping its timer then +// delegating to cancelCtx.cancel. +type timerCtx struct { + *cancelCtx + timer *time.Timer // Under cancelCtx.mu. + + deadline time.Time +} + +func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { + return c.deadline, true +} + +func (c *timerCtx) String() string { + return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) +} + +func (c *timerCtx) cancel(removeFromParent bool, err error) { + c.cancelCtx.cancel(false, err) + if removeFromParent { + // Remove this timerCtx from its parent cancelCtx's children. + removeChild(c.cancelCtx.Context, c) + } + c.mu.Lock() + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + c.mu.Unlock() +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return &valueCtx{parent, key, val} +} + +// A valueCtx carries a key-value pair. It implements Value for that key and +// delegates all other calls to the embedded Context. +type valueCtx struct { + Context + key, val interface{} +} + +func (c *valueCtx) String() string { + return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) +} + +func (c *valueCtx) Value(key interface{}) interface{} { + if c.key == key { + return c.val + } + return c.Context.Value(key) +} From 8c55e0c2d57ac92ac87a17166606dbc7c5395ad3 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Jul 2016 15:31:04 -0700 Subject: [PATCH 07/14] image/*/interface: Add unstable warnings to Engines As called for during today's meeting [1]. The wording is from Brandon [2]. [1]: http://ircbot.wl.linuxfoundation.org/meetings/opencontainers/2016/opencontainers.2016-07-27-21.00.log.html#l-54 [2]: /~https://github.com/opencontainers/image-spec/pull/159/files/9f8a18eb1cb2a1cbccd6fe137d743dccc2142901#r72525379 Signed-off-by: W. Trevor King --- image/cas/interface.go | 4 ++++ image/refs/interface.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/image/cas/interface.go b/image/cas/interface.go index 156ff48..58df8dd 100644 --- a/image/cas/interface.go +++ b/image/cas/interface.go @@ -22,6 +22,10 @@ import ( ) // Engine represents a content-addressable storage engine. +// +// This interface is for internal use of oci-image-tool for the time +// being. It is subject to change. This notice will be removed when +// and if the interface becomes stable. type Engine interface { // Put adds a new blob to the store. The action is idempotent; a diff --git a/image/refs/interface.go b/image/refs/interface.go index 92472eb..59d5b2c 100644 --- a/image/refs/interface.go +++ b/image/refs/interface.go @@ -25,6 +25,10 @@ import ( type ListNameCallback func(ctx context.Context, name string) (err error) // Engine represents a name-based reference storage engine. +// +// This interface is for internal use of oci-image-tool for the time +// being. It is subject to change. This notice will be removed when +// and if the interface becomes stable. type Engine interface { // Put adds a new reference to the store. The action is idempotent; From 7ea54f2bde8f7e78052f6674301c071be9c11671 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Jul 2016 15:16:10 -0700 Subject: [PATCH 08/14] image/cas: Implement Engine.Put This is a bit awkward. For writing a tar entry, we need to know both the name and size of the file ahead of time. The implementation in this commit accomplishes that by reading the Put content into a buffer, hashing and sizing the buffer, and then calling WriteTarEntryByName to create the entry. With a filesystem-backed CAS engine, we could avoid the buffer by writing the file to a temporary location with rolling hash and size tracking and then renaming the temporary file to the appropriate path. WriteTarEntryByName itself has awkward buffering to avoid dropping anything onto disk. It reads through its current file and writes the new tar into a buffer, and then writes that buffer back back over its current file. There are a few issues with this: * It's a lot more work than you need if you're just appending a new entry to the end of the tarball. But writing the whole file into a buffer means we don't have to worry about the trailing blocks that mark the end of the tarball; that's all handled transparently for us by the Go implementation. And this implementation doesn't have to be performant (folks should not be using tarballs to back write-heavy engines). * It could leave you with a corrupted tarball if the caller dies mid-overwrite. Again, I expect folks will only ever write to a tarball when building a tarball for publishing. If the caller dies, you can just start over. Folks looking for a more reliable implementation should use a filesystem-backed engine. * It could leave you with dangling bytes at the end of the tarball. I couldn't find a Go invocation to truncate the file. Go does have an ftruncate(2) wrapper [1], but it doesn't seem to be exposed at the io.Reader/io.Writer/... level. So if you write a shorter file with the same name as the original, you may end up with some dangling bytes. cas.Engine.Put protects against excessive writes with a Get guard; after hashing the new data, Put trys to Get it from the tarball and only writes a new entry if it can't find an existing entry. This also protects the CAS engine from the dangling-bytes issue. The 0666 file modes and 0777 directory modes rely on the caller's umask to appropriately limit user/group/other permissions for the tarball itself and any content extracted to the filesystem from the tarball. The trailing slash manipulation (stripping before comparison and injecting before creation) is based on part of libarchive's description of old-style archives [2]: name Pathname, stored as a null-terminated string. Early tar implementations only stored regular files (including hardlinks to those files). One common early convention used a trailing "/" character to indicate a directory name, allowing directory permissions and owner information to be archived and restored. and POSIX ustar archives [3]: name, prefix ... The standard does not require a trailing / character on directory names, though most implementations still include this for compatibility reasons. [1]: https://golang.org/pkg/syscall/#Ftruncate [2]: /~https://github.com/libarchive/libarchive/wiki/ManPageTar5#old-style-archive-format [3]: /~https://github.com/libarchive/libarchive/wiki/ManPageTar5#posix-ustar-archives Signed-off-by: W. Trevor King --- cmd/oci-cas/main.go | 1 + cmd/oci-cas/put.go | 83 +++++++++++++++++++++++ image/cas/layout/main.go | 2 +- image/cas/layout/tar.go | 36 +++++++++- image/layout/tar.go | 139 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 cmd/oci-cas/put.go diff --git a/cmd/oci-cas/main.go b/cmd/oci-cas/main.go index 6a604e3..c14f376 100644 --- a/cmd/oci-cas/main.go +++ b/cmd/oci-cas/main.go @@ -28,6 +28,7 @@ func main() { } cmd.AddCommand(newGetCmd()) + cmd.AddCommand(newPutCmd()) err := cmd.Execute() if err != nil { diff --git a/cmd/oci-cas/put.go b/cmd/oci-cas/put.go new file mode 100644 index 0000000..31abd2b --- /dev/null +++ b/cmd/oci-cas/put.go @@ -0,0 +1,83 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type putCmd struct { + path string +} + +func newPutCmd() *cobra.Command { + state := &putCmd{} + + return &cobra.Command{ + Use: "put PATH", + Short: "Write a blob to the store", + Long: "Read a blob from stdin, write it to the store, and print the digest to stdout.", + Run: state.Run, + } +} + +func (state *putCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *putCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + digest, err := engine.Put(ctx, os.Stdin) + if err != nil { + return err + } + + n, err := fmt.Fprintln(os.Stdout, digest) + if err != nil { + return err + } + if n < len(digest) { + return fmt.Errorf("wrote %d of %d bytes", n, len(digest)) + } + + return nil +} diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go index 1ab7b04..d9972a6 100644 --- a/image/cas/layout/main.go +++ b/image/cas/layout/main.go @@ -30,7 +30,7 @@ import ( // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) { - file, err := os.Open(path) + file, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { return nil, err } diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go index f4eab92..d8269e9 100644 --- a/image/cas/layout/tar.go +++ b/image/cas/layout/tar.go @@ -15,11 +15,14 @@ package layout import ( + "bytes" + "crypto/sha256" + "encoding/hex" "errors" + "fmt" "io" "io/ioutil" "os" - "strings" "github.com/opencontainers/image-tools/image/cas" "github.com/opencontainers/image-tools/image/layout" @@ -47,8 +50,35 @@ func NewTarEngine(ctx context.Context, file ReadWriteSeekCloser) (eng cas.Engine // Put adds a new blob to the store. func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { - // FIXME - return "", errors.New("TarEngine.Put is not supported yet") + data, err := ioutil.ReadAll(reader) + if err != nil { + return "", err + } + + size := int64(len(data)) + hash := sha256.Sum256(data) + hexHash := hex.EncodeToString(hash[:]) + algorithm := "sha256" + digest = fmt.Sprintf("%s:%s", algorithm, hexHash) + + _, err = engine.Get(ctx, digest) + if err == os.ErrNotExist { + var targetName string + targetName, err = blobPath(digest, "/") + if err != nil { + return "", err + } + + reader = bytes.NewReader(data) + err = layout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size) + if err != nil { + return "", err + } + } else if err != nil { + return "", err + } + + return digest, nil } // Get returns a reader for retrieving a blob from the store. diff --git a/image/layout/tar.go b/image/layout/tar.go index 270feba..1447a8e 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -16,11 +16,15 @@ package layout import ( "archive/tar" + "bytes" "encoding/json" "errors" "fmt" "io" + "io/ioutil" "os" + "strings" + "time" "golang.org/x/net/context" ) @@ -56,6 +60,141 @@ func TarEntryByName(ctx context.Context, reader io.ReadSeeker, name string) (hea } } +// WriteTarEntryByName reads content from reader into an entry at name +// in the tarball at file, replacing a previous entry with that name +// (if any). The current implementation avoids writing a temporary +// file to disk, but risks leaving a corrupted tarball if the program +// crashes mid-write. +// +// To add an entry to a tarball (with Go's interface) you need to know +// the size ahead of time. If you set the size argument, +// WriteTarEntryByName will use that size in the entry header (and +// Go's implementation will check to make sure it matches the length +// of content read from reader). If unset, WriteTarEntryByName will +// copy reader into a local buffer, measure its size, and then write +// the entry header and content. +func WriteTarEntryByName(ctx context.Context, file io.ReadWriteSeeker, name string, reader io.Reader, size *int64) (err error) { + var buffer bytes.Buffer + tarWriter := tar.NewWriter(&buffer) + + components := strings.Split(name, "/") + if components[0] != "." { + return fmt.Errorf("tar name entry does not start with './': %q", name) + } + + var parents []string + for i := 2; i < len(components); i++ { + parents = append(parents, strings.Join(components[:i], "/")) + } + + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + + tarReader := tar.NewReader(file) + found := false + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var header *tar.Header + header, err = tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + dirName := strings.TrimRight(header.Name, "/") + for i, parent := range parents { + if dirName == parent { + parents = append(parents[:i], parents[i+1:]...) + break + } + } + + if header.Name == name { + found = true + err = writeTarEntry(ctx, tarWriter, name, reader, size) + } else { + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + _, err = io.Copy(tarWriter, tarReader) + } + if err != nil { + return err + } + } + + if !found { + now := time.Now() + for _, parent := range parents { + header := &tar.Header{ + Name: parent + "/", + Mode: 0777, + ModTime: now, + Typeflag: tar.TypeDir, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + } + err = writeTarEntry(ctx, tarWriter, name, reader, size) + if err != nil { + return err + } + } + + err = tarWriter.Close() + if err != nil { + return err + } + + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + // FIXME: truncate file + + _, err = buffer.WriteTo(file) + return err +} + +func writeTarEntry(ctx context.Context, writer *tar.Writer, name string, reader io.Reader, size *int64) (err error) { + if size == nil { + var data []byte + data, err = ioutil.ReadAll(reader) + if err != nil { + return err + } + reader = bytes.NewReader(data) + _size := int64(len(data)) + size = &_size + } + now := time.Now() + header := &tar.Header{ + Name: name, + Mode: 0666, + Size: *size, + ModTime: now, + Typeflag: tar.TypeReg, + } + err = writer.WriteHeader(header) + if err != nil { + return err + } + + _, err = io.Copy(writer, reader) + return err +} + // CheckTarVersion walks a tarball pointed to by reader and returns an // error if oci-layout is missing or has unrecognized content. func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { From d2c026b32da45efc63661405f6fb6a2b5c680c55 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 28 Jul 2016 21:50:06 -0700 Subject: [PATCH 09/14] image/refs: Implement Engine.Put This is pretty straightforward with the new WriteTarEntryByName helper. I considered pulling the ref name -> path conversion (%s -> ./refs/%s) out into a helper function to stay DRY, but the logic is simple enough that it seemed more intuitive to leave it inline. Signed-off-by: W. Trevor King --- cmd/oci-refs/main.go | 1 + cmd/oci-refs/put.go | 81 +++++++++++++++++++++++++++++++++++++++ image/refs/layout/main.go | 2 +- image/refs/layout/tar.go | 15 +++++++- 4 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 cmd/oci-refs/put.go diff --git a/cmd/oci-refs/main.go b/cmd/oci-refs/main.go index cb2cc7d..35ea2c1 100644 --- a/cmd/oci-refs/main.go +++ b/cmd/oci-refs/main.go @@ -27,6 +27,7 @@ func main() { Short: "Name-based reference manipulation", } + cmd.AddCommand(newPutCmd()) cmd.AddCommand(newGetCmd()) cmd.AddCommand(newListCmd()) diff --git a/cmd/oci-refs/put.go b/cmd/oci-refs/put.go new file mode 100644 index 0000000..5d35e37 --- /dev/null +++ b/cmd/oci-refs/put.go @@ -0,0 +1,81 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type putCmd struct { + path string + name string +} + +func newPutCmd() *cobra.Command { + state := &putCmd{} + + return &cobra.Command{ + Use: "put PATH NAME", + Short: "Write a reference to the store", + Long: "Read descriptor JSON from stdin and write it to the store.", + Run: state.Run, + } +} + +func (state *putCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *putCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + decoder := json.NewDecoder(os.Stdin) + var descriptor specs.Descriptor + err = decoder.Decode(&descriptor) + if err != nil { + return err + } + + return engine.Put(ctx, state.name, &descriptor) +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go index 81f885c..d1722e5 100644 --- a/image/refs/layout/main.go +++ b/image/refs/layout/main.go @@ -29,7 +29,7 @@ import ( // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). func NewEngine(ctx context.Context, path string) (engine refs.Engine, err error) { - file, err := os.Open(path) + file, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { return nil, err } diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go index 12f903c..d23cd0e 100644 --- a/image/refs/layout/tar.go +++ b/image/refs/layout/tar.go @@ -16,6 +16,7 @@ package layout import ( "archive/tar" + "bytes" "encoding/json" "errors" "io" @@ -50,8 +51,18 @@ func NewTarEngine(ctx context.Context, file caslayout.ReadWriteSeekCloser) (eng // Put adds a new reference to the store. func (engine *TarEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { - // FIXME - return errors.New("TarEngine.Put is not supported yet") + data, err := json.Marshal(descriptor) + if err != nil { + return err + } + + size := int64(len(data)) + reader := bytes.NewReader(data) + targetName, err := refPath(name, "/") + if err != nil { + return err + } + return imagelayout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size) } // Get returns a reference from the store. From 5c748c9ae608b89ac99569802f0a53814728f1ba Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 28 Jul 2016 21:34:02 -0700 Subject: [PATCH 10/14] image/layout/tar: Add a CreateTarFile helper The NewEngine commands for the tar-backed image-layout engines (both the CAS and refs engines) open files O_RDWR and expect image-layout compatible content in the tarball. That makes sense, but for folks who *don't* have such a tarball, a helper like CreateTarFile makes it easy to explicitly create an empty one. The 0666 file modes and 0777 directory modes rely on the caller's umask to appropriately limit user/group/other permissions for the tarball itself and any content extracted to the filesystem from the tarball. The trailing slashes are based on part of libarchive's description of old-style archives [1]: name Pathname, stored as a null-terminated string. Early tar implementations only stored regular files (including hardlinks to those files). One common early convention used a trailing "/" character to indicate a directory name, allowing directory permissions and owner information to be archived and restored. and POSIX ustar archives [2]: name, prefix ... The standard does not require a trailing / character on directory names, though most implementations still include this for compatibility reasons. Expose this new functionality on the command line as: $ oci-image-init image-layout PATH where 'image-layout' is a separate level in case we support initializing additional types of repositories in the future. [1]: /~https://github.com/libarchive/libarchive/wiki/ManPageTar5#old-style-archive-format [2]: /~https://github.com/libarchive/libarchive/wiki/ManPageTar5#posix-ustar-archives Signed-off-by: W. Trevor King --- .gitignore | 1 + Makefile | 1 + cmd/oci-image-init/image_layout.go | 57 ++++++++++++++++++++++++++++++ cmd/oci-image-init/main.go | 37 +++++++++++++++++++ image/layout/tar.go | 47 ++++++++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 cmd/oci-image-init/image_layout.go create mode 100644 cmd/oci-image-init/main.go diff --git a/.gitignore b/.gitignore index fe4e8e5..144c59d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /oci-cas /oci-create-runtime-bundle +/oci-image-init /oci-image-validate /oci-refs /oci-unpack diff --git a/Makefile b/Makefile index ed26eae..c58a4b4 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ EPOCH_TEST_COMMIT ?= v0.2.0 TOOLS := \ oci-cas \ oci-create-runtime-bundle \ + oci-image-init \ oci-image-validate \ oci-refs \ oci-unpack diff --git a/cmd/oci-image-init/image_layout.go b/cmd/oci-image-init/image_layout.go new file mode 100644 index 0000000..10b8959 --- /dev/null +++ b/cmd/oci-image-init/image_layout.go @@ -0,0 +1,57 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type imageLayout struct{} + +func newImageLayoutCmd() *cobra.Command { + state := &imageLayout{} + + return &cobra.Command{ + Use: "image-layout PATH", + Short: "Initialize an OCI image-layout repository", + Run: state.Run, + } +} + +func (state *imageLayout) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + path := args[0] + + ctx := context.Background() + + err := layout.CreateTarFile(ctx, path) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/cmd/oci-image-init/main.go b/cmd/oci-image-init/main.go new file mode 100644 index 0000000..49f4aab --- /dev/null +++ b/cmd/oci-image-init/main.go @@ -0,0 +1,37 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + cmd := &cobra.Command{ + Use: "oci-image-init", + Short: "Initialize an OCI image", + } + + cmd.AddCommand(newImageLayoutCmd()) + + err := cmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/image/layout/tar.go b/image/layout/tar.go index 1447a8e..0b2ad85 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -218,3 +218,50 @@ func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { return nil } + +// CreateTarFile creates a new image-layout tar file at the given path. +func CreateTarFile(ctx context.Context, path string) (err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return err + } + defer file.Close() + + tarWriter := tar.NewWriter(file) + defer tarWriter.Close() + + now := time.Now() + for _, name := range []string{"./blobs/", "./refs/"} { + header := &tar.Header{ + Name: name, + Mode: 0777, + ModTime: now, + Typeflag: tar.TypeDir, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + } + + imageLayoutVersion := ImageLayoutVersion{ + Version: "1.0.0", + } + imageLayoutVersionBytes, err := json.Marshal(imageLayoutVersion) + if err != nil { + return err + } + header := &tar.Header{ + Name: "./oci-layout", + Mode: 0666, + Size: int64(len(imageLayoutVersionBytes)), + ModTime: now, + Typeflag: tar.TypeReg, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + _, err = tarWriter.Write(imageLayoutVersionBytes) + return err +} From 7bd8bcf3cd346e548bddc8a714c04f61c14d0d87 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 07:04:08 -0700 Subject: [PATCH 11/14] image: Refactor to use cas/ref engines instead of walkers The validation/unpacking code doesn't really care what the reference and CAS implemenations are. And the new generic interfaces in image/refs and image/cas will scale better as we add new backends than the walker interface. This replaces the simpler interface from image/reader.go with something more robust. The old tar/directory distinction between image and imageLayout is gone. The new CAS/refs engines don't support directory backends yet (I plan on adding them once the engine framework lands), but the new framework will handle tar/directory/... detection inside layout.NewEngine (and possibly inside a new (cas|refs).NewEngine when we grow engine types that aren't based on image-layout). Also replace the old methods like: func (d *descriptor) validateContent(r io.Reader) error with functions like: validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error to avoid local types that duplicate the image-spec types. This saves an extra instantiation for folks who want to validate (or whatever) a specs.Descriptor they have obtained elsewhere. I'd prefer casLayout and refsLayout for the imported packages, but Stephen doesn't want camelCase for package names [1]. [1]: /~https://github.com/opencontainers/image-spec/pull/159#discussion_r76720225 Signed-off-by: W. Trevor King --- cmd/oci-create-runtime-bundle/main.go | 9 +- .../oci-create-runtime-bundle.1.md | 2 +- cmd/oci-image-validate/main.go | 14 +- .../oci-image-validate.1.md | 6 +- cmd/oci-unpack/main.go | 13 +- cmd/oci-unpack/oci-unpack.1.md | 2 +- image/autodetect.go | 3 +- image/config.go | 81 +++---- image/descriptor.go | 99 ++------ image/image.go | 201 +++++++++------- image/image_test.go | 217 +++++------------- image/manifest.go | 125 +++++----- image/manifest_test.go | 99 +++----- image/reader.go | 84 ------- image/walker.go | 121 ---------- 15 files changed, 341 insertions(+), 735 deletions(-) delete mode 100644 image/reader.go delete mode 100644 image/walker.go diff --git a/cmd/oci-create-runtime-bundle/main.go b/cmd/oci-create-runtime-bundle/main.go index c517d09..f76563c 100644 --- a/cmd/oci-create-runtime-bundle/main.go +++ b/cmd/oci-create-runtime-bundle/main.go @@ -23,6 +23,7 @@ import ( specs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-tools/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // gitCommit will be the hash that the binary was built from @@ -31,7 +32,6 @@ var gitCommit = "" // supported bundle types var bundleTypes = []string{ - image.TypeImageLayout, image.TypeImage, } @@ -109,6 +109,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if _, err := os.Stat(args[1]); os.IsNotExist(err) { v.stderr.Printf("destination path %s does not exist", args[1]) os.Exit(1) @@ -125,11 +127,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case image.TypeImageLayout: - err = image.CreateRuntimeBundleLayout(args[0], args[1], v.ref, v.root) - case image.TypeImage: - err = image.CreateRuntimeBundle(args[0], args[1], v.ref, v.root) + err = image.CreateRuntimeBundle(ctx, args[0], args[1], v.ref, v.root) } if err != nil { diff --git a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md index ad95181..8fee487 100644 --- a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md +++ b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md @@ -24,7 +24,7 @@ runtime-spec-compatible `dest/config.json`. A directory representing the root filesystem of the container in the OCI runtime bundle. It is strongly recommended to keep the default value. (default "rootfs") **--type** - Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "image" # EXAMPLES ``` diff --git a/cmd/oci-image-validate/main.go b/cmd/oci-image-validate/main.go index 5143b63..0fc0978 100644 --- a/cmd/oci-image-validate/main.go +++ b/cmd/oci-image-validate/main.go @@ -25,6 +25,7 @@ import ( "github.com/opencontainers/image-tools/image" "github.com/pkg/errors" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // gitCommit will be the hash that the binary was built from @@ -33,7 +34,6 @@ var gitCommit = "" // supported validation types var validateTypes = []string{ - image.TypeImageLayout, image.TypeImage, image.TypeManifest, image.TypeManifestList, @@ -81,7 +81,7 @@ func newValidateCmd(stdout, stderr *log.Logger) *cobra.Command { cmd.Flags().StringSliceVar( &v.refs, "ref", nil, - `A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image or imageLayout.`, + `A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image.`, ) cmd.Flags().BoolVar( @@ -107,9 +107,11 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + var exitcode int for _, arg := range args { - err := v.validatePath(arg) + err := v.validatePath(ctx, arg) if err == nil { v.stdout.Printf("%s: OK", arg) @@ -139,7 +141,7 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(exitcode) } -func (v *validateCmd) validatePath(name string) error { +func (v *validateCmd) validatePath(ctx context.Context, name string) error { var ( err error typ = v.typ @@ -152,10 +154,8 @@ func (v *validateCmd) validatePath(name string) error { } switch typ { - case image.TypeImageLayout: - return image.ValidateLayout(name, v.refs, v.stdout) case image.TypeImage: - return image.Validate(name, v.refs, v.stdout) + return image.Validate(ctx, name, v.refs, v.stdout) } f, err := os.Open(name) diff --git a/cmd/oci-image-validate/oci-image-validate.1.md b/cmd/oci-image-validate/oci-image-validate.1.md index 33d2b49..cf04249 100644 --- a/cmd/oci-image-validate/oci-image-validate.1.md +++ b/cmd/oci-image-validate/oci-image-validate.1.md @@ -20,15 +20,15 @@ oci-image-validate \- Validate one or more image files Can be specified multiple times to validate multiple references. `NAME` must be present in the `refs` subdirectory of the image. Defaults to `v1.0`. - Only applicable if type is image or imageLayout. + Only applicable if type is image. **--type** - Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "imageLayout,image,manifest,manifestList,config" + Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "image,manifest,manifestList,config" # EXAMPLES ``` $ skopeo copy docker://busybox oci:busybox-oci -$ oci-image-validate --type imageLayout --ref latest busybox-oci +$ oci-image-validate --type image --ref latest busybox-oci busybox-oci: OK ``` diff --git a/cmd/oci-unpack/main.go b/cmd/oci-unpack/main.go index ed0ffa7..e754d50 100644 --- a/cmd/oci-unpack/main.go +++ b/cmd/oci-unpack/main.go @@ -23,6 +23,7 @@ import ( specs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-tools/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // gitCommit will be the hash that the binary was built from @@ -31,7 +32,6 @@ var gitCommit = "" // supported unpack types var unpackTypes = []string{ - image.TypeImageLayout, image.TypeImage, } @@ -62,8 +62,8 @@ func newUnpackCmd(stdout, stderr *log.Logger) *cobra.Command { cmd := &cobra.Command{ Use: "unpack [src] [dest]", - Short: "Unpack an image or image source layout", - Long: `Unpack the OCI image .tar file or OCI image layout directory present at [src] to the destination directory [dest].`, + Short: "Unpack an image", + Long: `Unpack the OCI image present at [src] to the destination directory [dest].`, Run: v.Run, } @@ -101,6 +101,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if v.typ == "" { typ, err := image.Autodetect(args[0]) if err != nil { @@ -112,11 +114,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case image.TypeImageLayout: - err = image.UnpackLayout(args[0], args[1], v.ref) - case image.TypeImage: - err = image.Unpack(args[0], args[1], v.ref) + err = image.Unpack(ctx, args[0], args[1], v.ref) } if err != nil { diff --git a/cmd/oci-unpack/oci-unpack.1.md b/cmd/oci-unpack/oci-unpack.1.md index b7529f9..4e9956d 100644 --- a/cmd/oci-unpack/oci-unpack.1.md +++ b/cmd/oci-unpack/oci-unpack.1.md @@ -18,7 +18,7 @@ oci-unpack \- Unpack an image or image source layout The ref pointing to the manifest to be unpacked. This must be present in the "refs" subdirectory of the image. (default "v1.0") **--type** - Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "image" # EXAMPLES ``` diff --git a/image/autodetect.go b/image/autodetect.go index c3e2f85..5b0552e 100644 --- a/image/autodetect.go +++ b/image/autodetect.go @@ -27,7 +27,6 @@ import ( // supported autodetection types const ( - TypeImageLayout = "imageLayout" TypeImage = "image" TypeManifest = "manifest" TypeManifestList = "manifestList" @@ -43,7 +42,7 @@ func Autodetect(path string) (string, error) { } if fi.IsDir() { - return TypeImageLayout, nil + return TypeImage, nil } f, err := os.Open(path) diff --git a/image/config.go b/image/config.go index 100d99c..81daea9 100644 --- a/image/config.go +++ b/image/config.go @@ -18,63 +18,64 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" - "os" - "path/filepath" "strconv" "strings" "github.com/opencontainers/image-spec/schema" + imagespecs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/image-tools/image/cas" + runtimespecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type config v1.Image +func findConfig(ctx context.Context, engine cas.Engine, descriptor *imagespecs.Descriptor) (config *v1.Image, err error) { + err = validateMediaType(descriptor.MediaType, []string{v1.MediaTypeImageConfig}) + if err != nil { + return nil, errors.Wrap(err, "invalid config media type") + } -func findConfig(w walker, d *descriptor) (*config, error) { - var c config - cpath := filepath.Join("blobs", d.algo(), d.hash()) + err = validateDescriptor(ctx, engine, descriptor) + if err != nil { + return nil, errors.Wrap(err, "invalid config descriptor") + } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != cpath { - return nil - } - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading config", path) - } + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) + } - if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: config validation failed", path) - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if err := json.Unmarshal(buf, &c); err != nil { - return err - } - // check if the rootfs type is 'layers' - if c.RootFS.Type != "layers" { - return fmt.Errorf("%q is an unknown rootfs type, MUST be 'layers'", c.RootFS.Type) - } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: config not found", cpath) - case errEOW: - return &c, nil - default: + if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: config validation failed", descriptor.Digest) + } + + var c v1.Image + if err := json.Unmarshal(buf, &c); err != nil { return nil, err } + + // check if the rootfs type is 'layers' + if c.RootFS.Type != "layers" { + return nil, fmt.Errorf("%q is an unknown rootfs type, MUST be 'layers'", c.RootFS.Type) + } + + return &c, nil } -func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { +func runtimeSpec(c *v1.Image, rootfs string) (*runtimespecs.Spec, error) { if c.OS != "linux" { return nil, fmt.Errorf("%s: unsupported OS", c.OS) } - var s specs.Spec - s.Version = specs.Version + var s runtimespecs.Spec + s.Version = runtimespecs.Version // we should at least apply the default spec, otherwise this is totally useless s.Process.Terminal = true s.Root.Path = rootfs @@ -116,12 +117,12 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { swap := uint64(c.Config.MemorySwap) shares := uint64(c.Config.CPUShares) - s.Linux.Resources = &specs.Resources{ - CPU: &specs.CPU{ + s.Linux.Resources = &runtimespecs.Resources{ + CPU: &runtimespecs.CPU{ Shares: &shares, }, - Memory: &specs.Memory{ + Memory: &runtimespecs.Memory{ Limit: &mem, Reservation: &mem, Swap: &swap, @@ -131,7 +132,7 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { for vol := range c.Config.Volumes { s.Mounts = append( s.Mounts, - specs.Mount{ + runtimespecs.Mount{ Destination: vol, Type: "bind", Options: []string{"rbind"}, diff --git a/image/descriptor.go b/image/descriptor.go index 1e760a7..73eaf89 100644 --- a/image/descriptor.go +++ b/image/descriptor.go @@ -17,105 +17,34 @@ package image import ( "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" - "os" - "path/filepath" - "strings" + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/cas" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type descriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -func (d *descriptor) algo() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[0] -} - -func (d *descriptor) hash() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[1] -} - -func listReferences(w walker) (map[string]*descriptor, error) { - refs := make(map[string]*descriptor) - - if err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || !strings.HasPrefix(path, "refs") { +func validateMediaType(mediaType string, mediaTypes []string) error { + for _, mt := range mediaTypes { + if mt == mediaType { return nil } - - var d descriptor - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - refs[info.Name()] = &d - - return nil - }); err != nil { - return nil, err } - return refs, nil + return fmt.Errorf("invalid media type %q", mediaType) } -func findDescriptor(w walker, name string) (*descriptor, error) { - var d descriptor - dpath := filepath.Join("refs", name) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != dpath { - return nil - } - - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: descriptor not found", dpath) - case errEOW: - return &d, nil - default: - return nil, err - } -} - -func (d *descriptor) validate(w walker, mts []string) error { - var found bool - for _, mt := range mts { - if d.MediaType == mt { - found = true - break - } - } - if !found { - return fmt.Errorf("invalid descriptor MediaType %q", d.MediaType) - } - - rc, err := w.Get(*d) +func validateDescriptor(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) error { + reader, err := engine.Get(ctx, descriptor.Digest) if err != nil { - return err + return errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) } - defer rc.Close() - return d.validateContent(rc) + return validateContent(ctx, descriptor, reader) } -func (d *descriptor) validateContent(r io.Reader) error { +func validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error { h := sha256.New() n, err := io.Copy(h, r) if err != nil { @@ -124,11 +53,11 @@ func (d *descriptor) validateContent(r io.Reader) error { digest := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if digest != d.Digest { + if digest != descriptor.Digest { return errors.New("digest mismatch") } - if n != d.Size { + if n != descriptor.Size { return errors.New("size mismatch") } diff --git a/image/image.go b/image/image.go index 1551c1c..70d7019 100644 --- a/image/image.go +++ b/image/image.go @@ -16,176 +16,203 @@ package image import ( "encoding/json" - "fmt" "log" "os" "path/filepath" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/image-tools/image/cas" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/opencontainers/image-tools/image/refs" + refslayout "github.com/opencontainers/image-tools/image/refs/layout" "github.com/pkg/errors" + "golang.org/x/net/context" ) -// ValidateLayout walks through the given file tree and validates the manifest -// pointed to by the given refs or returns an error if the validation failed. -func ValidateLayout(src string, refs []string, out *log.Logger) error { - return validate(newPathWalker(src), refs, out) +var validRefMediaTypes = []string{ + v1.MediaTypeImageManifest, + v1.MediaTypeImageManifestList, } -// Validate walks through the given .tar file and validates the manifest -// pointed to by the given refs or returns an error if the validation failed. -func Validate(tarFile string, refs []string, out *log.Logger) error { - f, err := os.Open(tarFile) +// Validate validates the given reference. +func Validate(ctx context.Context, path string, refs []string, out *log.Logger) error { + refEngine, err := refslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer refEngine.Close() - return validate(newTarWalker(tarFile, f), refs, out) -} + casEngine, err := caslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer casEngine.Close() -var validRefMediaTypes = []string{ - v1.MediaTypeImageManifest, - v1.MediaTypeImageManifestList, -} + if len(refs) > 0 { + for _, ref := range refs { + err = validate(ctx, refEngine, casEngine, ref, out) + if err != nil { + return err + } + } + } -func validate(w walker, refs []string, out *log.Logger) error { - ds, err := listReferences(w) + count := 0 + err = refEngine.List( + ctx, + "", + -1, + 0, + func(ctx context.Context, name string) error { + count++ + return validate(ctx, refEngine, casEngine, name, out) + }, + ) if err != nil { return err } - if len(refs) == 0 && len(ds) == 0 { + + if count == 0 { // TODO(runcom): ugly, we'll need a better way and library // to express log levels. // see /~https://github.com/opencontainers/image-spec/issues/288 out.Print("WARNING: no descriptors found") } - if len(refs) == 0 { - for ref := range ds { - refs = append(refs, ref) - } + return nil +} + +func validate(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, ref string, out *log.Logger) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) } - for _, ref := range refs { - d, ok := ds[ref] - if !ok { - // TODO(runcom): - // soften this error to a warning if the user didn't ask for any specific reference - // with --ref but she's just validating the whole image. - return fmt.Errorf("reference %s not found", ref) - } + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) + if err != nil { + return err + } - if err = d.validate(w, validRefMediaTypes); err != nil { - return err - } + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { + return err + } - m, err := findManifest(w, d) - if err != nil { - return err - } + m, err := findManifest(ctx, casEngine, descriptor) + if err != nil { + return err + } - if err := m.validate(w); err != nil { - return err - } - if out != nil { - out.Printf("reference %q: OK", ref) - } + err = validateManifest(ctx, m, casEngine) + if err != nil { + return err + } + + if out != nil { + out.Printf("reference %q: OK", ref) } return nil } -// UnpackLayout walks through the file tree given by src and, using the layers -// specified in the manifest pointed to by the given ref, unpacks all layers in -// the given destination directory or returns an error if the unpacking failed. -func UnpackLayout(src, dest, ref string) error { - return unpack(newPathWalker(src), dest, ref) -} +// Unpack unpacks the given reference to a destination directory. +func Unpack(ctx context.Context, path, dest, ref string) error { + refEngine, err := refslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer refEngine.Close() -// Unpack walks through the given .tar file and, using the layers specified in -// the manifest pointed to by the given ref, unpacks all layers in the given -// destination directory or returns an error if the unpacking failed. -func Unpack(tarFile, dest, ref string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return unpack(newTarWalker(tarFile, f), dest, ref) + return unpack(ctx, refEngine, casEngine, dest, ref) } -func unpack(w walker, dest, refName string) error { - ref, err := findDescriptor(w, refName) +func unpack(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref string) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) + } + + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) if err != nil { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = validateManifest(ctx, m, casEngine); err != nil { return err } - return m.unpack(w, dest) + return unpackManifest(ctx, m, casEngine, dest) } -// CreateRuntimeBundleLayout walks through the file tree given by src and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundleLayout(src, dest, ref, root string) error { - return createRuntimeBundle(newPathWalker(src), dest, ref, root) -} +// CreateRuntimeBundle creates an OCI runtime bundle in the given +// destination. +func CreateRuntimeBundle(ctx context.Context, path, dest, ref, rootfs string) error { + refEngine, err := refslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer refEngine.Close() -// CreateRuntimeBundle walks through the given .tar file and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundle(tarFile, dest, ref, root string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return createRuntimeBundle(newTarWalker(tarFile, f), dest, ref, root) + return createRuntimeBundle(ctx, refEngine, casEngine, dest, ref, rootfs) } -func createRuntimeBundle(w walker, dest, refName, rootfs string) error { - ref, err := findDescriptor(w, refName) +func createRuntimeBundle(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref, rootfs string) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) + } + + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) if err != nil { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = validateManifest(ctx, m, casEngine); err != nil { return err } - c, err := findConfig(w, &m.Config) + c, err := findConfig(ctx, casEngine, &m.Config) if err != nil { return err } - err = m.unpack(w, filepath.Join(dest, rootfs)) + err = unpackManifest(ctx, m, casEngine, filepath.Join(dest, rootfs)) if err != nil { return err } - spec, err := c.runtimeSpec(rootfs) + spec, err := runtimeSpec(c, rootfs) if err != nil { return err } diff --git a/image/image_test.go b/image/image_test.go index 8c3c0c9..fec077d 100644 --- a/image/image_test.go +++ b/image/image_test.go @@ -18,24 +18,25 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha256" - "fmt" "io" "io/ioutil" "os" - "path/filepath" "strconv" "strings" "testing" + "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" + cas "github.com/opencontainers/image-tools/image/cas" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" + refslayout "github.com/opencontainers/image-tools/image/refs/layout" + "golang.org/x/net/context" ) const ( refTag = "latest" - layoutStr = `{"imageLayoutVersion": "1.0.0"}` - configStr = `{ "created": "2015-10-31T22:22:56.015925234Z", "author": "Alyssa P. Hacker ", @@ -91,8 +92,6 @@ const ( ) var ( - refStr = `{"digest":"","mediaType":"application/vnd.oci.image.manifest.v1+json","size":}` - manifestStr = `{ "annotations": null, "config": { @@ -118,214 +117,106 @@ type tarContent struct { b []byte } -type imageLayout struct { - rootDir string - layout string - ref string - manifest string - config string - tarList []tarContent -} - func TestValidateLayout(t *testing.T) { + ctx := context.Background() + root, err := ioutil.TempDir("", "oci-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(root) - il := imageLayout{ - rootDir: root, - layout: layoutStr, - ref: refTag, - manifest: manifestStr, - config: configStr, - tarList: []tarContent{ - tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, - }, - } - - // create image layout bundle - err = createImageLayoutBundle(il) + err = imagelayout.CreateDir(ctx, root) if err != nil { t.Fatal(err) } - err = ValidateLayout(root, []string{refTag}, nil) + casEngine, err := caslayout.NewEngine(ctx, root) if err != nil { t.Fatal(err) } -} - -func createImageLayoutBundle(il imageLayout) error { - err := os.MkdirAll(filepath.Join(il.rootDir, "blobs", "sha256"), 0700) - if err != nil { - return err - } - - err = os.MkdirAll(filepath.Join(il.rootDir, "refs"), 0700) - if err != nil { - return err - } - - // create image layout file - err = createLayoutFile(il.rootDir) - if err != nil { - return err - } - - // create image layer blob file. - desc, err := createImageLayerFile(il.rootDir, il.tarList) - if err != nil { - return err - } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) - il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) - - desc, err = createConfigFile(il.rootDir, il.config) - if err != nil { - return err - } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) - il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) - - // create manifest blob file - desc, err = createManifestFile(il.rootDir, il.manifest) - if err != nil { - return err - } - - return createRefFile(il.rootDir, il.ref, desc) -} + defer casEngine.Close() -func createLayoutFile(root string) error { - layoutPath := filepath.Join(root, "oci-layout") - f, err := os.Create(layoutPath) + refsEngine, err := refslayout.NewEngine(ctx, root) if err != nil { - return err - } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(layoutStr))) - return err -} - -func createRefFile(root, ref string, mft descriptor) error { - refpath := filepath.Join(root, "refs", ref) - f, err := os.Create(refpath) - if err != nil { - return err + t.Fatal(err) } - defer f.Close() - refStr = strings.Replace(refStr, "", mft.Digest, -1) - refStr = strings.Replace(refStr, "", strconv.FormatInt(mft.Size, 10), -1) - _, err = io.Copy(f, bytes.NewBuffer([]byte(refStr))) - return err -} + defer refsEngine.Close() -func createManifestFile(root, str string) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-manifest") - f, err := os.Create(name) + layer, err := createTarBlob(ctx, casEngine, []tarContent{ + tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, + }) if err != nil { - return descriptor{}, err + t.Fatal(err) } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(str))) - if err != nil { - return descriptor{}, err + digest, err := casEngine.Put(ctx, strings.NewReader(configStr)) + config := specs.Descriptor{ + Digest: digest, + MediaType: v1.MediaTypeImageConfig, + Size: int64(len(configStr)), } - return createHashedBlob(name) -} - -func createConfigFile(root, config string) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-config") - f, err := os.Create(name) - if err != nil { - return descriptor{}, err + _manifest := manifestStr + _manifest = strings.Replace(_manifest, "", config.Digest, 1) + _manifest = strings.Replace(_manifest, "", strconv.FormatInt(config.Size, 10), 1) + _manifest = strings.Replace(_manifest, "", layer.Digest, 1) + _manifest = strings.Replace(_manifest, "", strconv.FormatInt(layer.Size, 10), 1) + digest, err = casEngine.Put(ctx, strings.NewReader(_manifest)) + manifest := specs.Descriptor{ + Digest: digest, + MediaType: v1.MediaTypeImageManifest, + Size: int64(len(_manifest)), } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(config))) + err = refsEngine.Put(ctx, refTag, &manifest) if err != nil { - return descriptor{}, err - } - - return createHashedBlob(name) -} - -func createImageLayerFile(root string, list []tarContent) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-layer") - err := createTarBlob(name, list) - if err != nil { - return descriptor{}, err + t.Fatal(err) } - desc, err := createHashedBlob(name) + err = Validate(ctx, root, []string{refTag}, nil) if err != nil { - return descriptor{}, err + t.Fatal(err) } - - desc.MediaType = v1.MediaTypeImageLayer - return desc, nil } -func createTarBlob(name string, list []tarContent) error { - file, err := os.Create(name) - if err != nil { - return err - } - defer file.Close() - gzipWriter := gzip.NewWriter(file) +func createTarBlob(ctx context.Context, engine cas.Engine, list []tarContent) (descriptor *specs.Descriptor, err error) { + var buffer bytes.Buffer + gzipWriter := gzip.NewWriter(&buffer) defer gzipWriter.Close() tarWriter := tar.NewWriter(gzipWriter) defer tarWriter.Close() for _, content := range list { if err = tarWriter.WriteHeader(content.header); err != nil { - return err + return nil, err } if _, err = io.Copy(tarWriter, bytes.NewReader(content.b)); err != nil { - return err + return nil, err } } - return nil -} -func createHashedBlob(name string) (descriptor, error) { - desc, err := newDescriptor(name) + err = tarWriter.Close() if err != nil { - return descriptor{}, err + return nil, err } - // Rename the file to hashed-digest name. - err = os.Rename(name, filepath.Join(filepath.Dir(name), desc.Digest)) + err = gzipWriter.Close() if err != nil { - return descriptor{}, err + return nil, err } - //Normalize the hashed digest. - desc.Digest = "sha256:" + desc.Digest - - return desc, nil -} - -func newDescriptor(name string) (descriptor, error) { - file, err := os.Open(name) - if err != nil { - return descriptor{}, err + var desc = specs.Descriptor{ + MediaType: v1.MediaTypeImageLayer, + Size: int64(buffer.Len()), } - defer file.Close() - // generate sha256 hash - hash := sha256.New() - size, err := io.Copy(hash, file) + digest, err := engine.Put(ctx, &buffer) if err != nil { - return descriptor{}, err + return nil, err } - return descriptor{ - Digest: fmt.Sprintf("%x", hash.Sum(nil)), - Size: size, - }, nil + desc.Digest = digest + + return &desc, nil } diff --git a/image/manifest.go b/image/manifest.go index c438ede..fb41b04 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -28,67 +28,67 @@ import ( "time" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/image-tools/image/cas" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type manifest struct { - Config descriptor `json:"config"` - Layers []descriptor `json:"layers"` -} - -func findManifest(w walker, d *descriptor) (*manifest, error) { - var m manifest - mpath := filepath.Join("blobs", d.algo(), d.hash()) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != mpath { - return nil - } - - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading manifest", path) - } - - if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: manifest validation failed", path) - } +func findManifest(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) (*v1.Manifest, error) { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) + } - if err := json.Unmarshal(buf, &m); err != nil { - return err - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if len(m.Layers) == 0 { - return fmt.Errorf("%s: no layers found", path) - } + if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: manifest validation failed", descriptor.Digest) + } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: manifest not found", mpath) - case errEOW: - return &m, nil - default: + var m v1.Manifest + if err := json.Unmarshal(buf, &m); err != nil { return nil, err } + + if len(m.Layers) == 0 { + return nil, fmt.Errorf("%s: no layers found", descriptor.Digest) + } + + return &m, nil } -func (m *manifest) validate(w walker) error { - if err := m.Config.validate(w, []string{v1.MediaTypeImageConfig}); err != nil { - return errors.Wrap(err, "config validation failed") +func validateManifest(ctx context.Context, m *v1.Manifest, engine cas.Engine) error { + _, err := findConfig(ctx, engine, &m.Config) + if err != nil { + return errors.Wrap(err, "invalid manifest config") } for _, d := range m.Layers { - if err := d.validate(w, []string{v1.MediaTypeImageLayer}); err != nil { - return errors.Wrap(err, "layer validation failed") + err = validateMediaType( + d.MediaType, + []string{ + v1.MediaTypeImageLayer, + v1.MediaTypeImageLayerNonDistributable, + }, + ) + if err != nil { + return errors.Wrap(err, "invalid layer media type") + } + err = validateDescriptor(ctx, engine, &d) + if err != nil { + return errors.Wrap(err, "invalid layer descriptor") } } return nil } -func (m *manifest) unpack(w walker, dest string) (retErr error) { +func unpackManifest(ctx context.Context, m *v1.Manifest, engine cas.Engine, dest string) (err error) { // error out if the dest directory is not empty s, err := ioutil.ReadDir(dest) if err != nil && !os.IsNotExist(err) { @@ -100,38 +100,33 @@ func (m *manifest) unpack(w walker, dest string) (retErr error) { defer func() { // if we encounter error during unpacking // clean up the partially-unpacked destination - if retErr != nil { - if err := os.RemoveAll(dest); err != nil { + if err != nil { + err2 := os.RemoveAll(dest) + if err2 != nil { fmt.Printf("Error: failed to remove partially-unpacked destination %v", err) } } }() + for _, d := range m.Layers { - if d.MediaType != string(schema.MediaTypeImageLayer) { - continue + err = validateMediaType( + d.MediaType, + []string{ + v1.MediaTypeImageLayer, + v1.MediaTypeImageLayerNonDistributable, + }, + ) + if err != nil { + return errors.Wrap(err, "invalid layer media type") } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() { - return nil - } - - dd, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != dd { - return nil - } - - if err := unpackLayer(dest, r); err != nil { - return errors.Wrap(err, "error extracting layer") - } + reader, err := engine.Get(ctx, d.Digest) + if err != nil { + return errors.Wrapf(err, "failed to fetch %s", d.Digest) + } - return errEOW - }); err { - case nil: - return fmt.Errorf("%s: layer not found", dest) - case errEOW: - default: - return err + if err := unpackLayer(dest, reader); err != nil { + return errors.Wrap(err, "error extracting layer") } } return nil diff --git a/image/manifest_test.go b/image/manifest_test.go index f50b2ed..96b1cdb 100644 --- a/image/manifest_test.go +++ b/image/manifest_test.go @@ -18,14 +18,18 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha256" - "fmt" "io" "io/ioutil" "os" "path/filepath" "strings" "testing" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-spec/specs-go/v1" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" + "golang.org/x/net/context" ) func TestUnpackLayerDuplicateEntries(t *testing.T) { @@ -66,53 +70,38 @@ func TestUnpackLayerDuplicateEntries(t *testing.T) { } func TestUnpackLayer(t *testing.T) { + ctx := context.Background() + tmp1, err := ioutil.TempDir("", "test-layer") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmp1) - err = os.MkdirAll(filepath.Join(tmp1, "blobs", "sha256"), 0700) - if err != nil { - t.Fatal(err) - } - tarfile := filepath.Join(tmp1, "blobs", "sha256", "test.tar") - f, err := os.Create(tarfile) - if err != nil { - t.Fatal(err) - } - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - tw.WriteHeader(&tar.Header{Name: "test", Size: 4, Mode: 0600}) - io.Copy(tw, bytes.NewReader([]byte("test"))) - tw.Close() - gw.Close() - f.Close() - // generate sha256 hash - h := sha256.New() - file, err := os.Open(tarfile) + path := filepath.Join(tmp1, "image.tar") + err = imagelayout.CreateTarFile(ctx, path) if err != nil { t.Fatal(err) } - defer file.Close() - _, err = io.Copy(h, file) + + engine, err := caslayout.NewEngine(ctx, path) if err != nil { t.Fatal(err) } - err = os.Rename(tarfile, filepath.Join(tmp1, "blobs", "sha256", fmt.Sprintf("%x", h.Sum(nil)))) + defer engine.Close() + + layer, err := createTarBlob(ctx, engine, []tarContent{ + tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, + }) if err != nil { t.Fatal(err) } - testManifest := manifest{ - Layers: []descriptor{descriptor{ - MediaType: "application/vnd.oci.image.layer.tar+gzip", - Digest: fmt.Sprintf("sha256:%s", fmt.Sprintf("%x", h.Sum(nil))), - }}, + testManifest := v1.Manifest{ + Layers: []specs.Descriptor{*layer}, } - err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) + + err = unpackManifest(ctx, &testManifest, engine, filepath.Join(tmp1, "rootfs")) if err != nil { t.Fatal(err) } @@ -124,56 +113,38 @@ func TestUnpackLayer(t *testing.T) { } func TestUnpackLayerRemovePartialyUnpackedFile(t *testing.T) { + ctx := context.Background() + // generate a tar file has duplicate entry which will failed on unpacking tmp1, err := ioutil.TempDir("", "test-layer") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmp1) - err = os.MkdirAll(filepath.Join(tmp1, "blobs", "sha256"), 0700) - if err != nil { - t.Fatal(err) - } - tarfile := filepath.Join(tmp1, "blobs", "sha256", "test.tar") - f, err := os.Create(tarfile) - if err != nil { - t.Fatal(err) - } - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - tw.WriteHeader(&tar.Header{Name: "test", Size: 4, Mode: 0600}) - io.Copy(tw, bytes.NewReader([]byte("test"))) - tw.WriteHeader(&tar.Header{Name: "test", Size: 5, Mode: 0600}) - io.Copy(tw, bytes.NewReader([]byte("test1"))) - tw.Close() - gw.Close() - f.Close() - // generate sha256 hash - h := sha256.New() - file, err := os.Open(tarfile) + err = imagelayout.CreateDir(ctx, tmp1) if err != nil { t.Fatal(err) } - defer file.Close() - _, err = io.Copy(h, file) + + engine, err := caslayout.NewEngine(ctx, tmp1) if err != nil { t.Fatal(err) } - err = os.Rename(tarfile, filepath.Join(tmp1, "blobs", "sha256", fmt.Sprintf("%x", h.Sum(nil)))) + defer engine.Close() + + layer, err := createTarBlob(ctx, engine, []tarContent{ + tarContent{&tar.Header{Name: "test", Size: 4, Mode: 0600}, []byte("test")}, + tarContent{&tar.Header{Name: "test", Size: 5, Mode: 0600}, []byte("test1")}, + }) if err != nil { t.Fatal(err) } - testManifest := manifest{ - Layers: []descriptor{descriptor{ - MediaType: "application/vnd.oci.image.layer.tar+gzip", - Digest: fmt.Sprintf("sha256:%s", fmt.Sprintf("%x", h.Sum(nil))), - }}, + testManifest := v1.Manifest{ + Layers: []specs.Descriptor{*layer}, } - err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) + err = unpackManifest(ctx, &testManifest, engine, filepath.Join(tmp1, "rootfs")) if err != nil && !strings.Contains(err.Error(), "duplicate entry for") { t.Fatal(err) } diff --git a/image/reader.go b/image/reader.go deleted file mode 100644 index 078db5a..0000000 --- a/image/reader.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2016 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" -) - -type reader interface { - Get(desc descriptor) (io.ReadCloser, error) -} - -type tarReader struct { - name string -} - -func (r *tarReader) Get(desc descriptor) (io.ReadCloser, error) { - f, err := os.Open(r.name) - if err != nil { - return nil, err - } - defer f.Close() - - tr := tar.NewReader(f) -loop: - for { - hdr, err := tr.Next() - switch err { - case io.EOF: - break loop - case nil: - // success, continue below - default: - return nil, err - } - if hdr.Name == filepath.Join("blobs", desc.algo(), desc.hash()) && - !hdr.FileInfo().IsDir() { - buf, err := ioutil.ReadAll(tr) - if err != nil { - return nil, err - } - return ioutil.NopCloser(bytes.NewReader(buf)), nil - } - } - - return nil, fmt.Errorf("object not found") -} - -type layoutReader struct { - root string -} - -func (r *layoutReader) Get(desc descriptor) (io.ReadCloser, error) { - name := filepath.Join(r.root, "blobs", desc.algo(), desc.hash()) - - info, err := os.Stat(name) - if err != nil { - return nil, err - } - - if info.IsDir() { - return nil, fmt.Errorf("object is dir") - } - - return os.Open(name) -} diff --git a/image/walker.go b/image/walker.go deleted file mode 100644 index 27acfac..0000000 --- a/image/walker.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2016 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -var ( - errEOW = fmt.Errorf("end of walk") // error to signal stop walking -) - -// walkFunc is a function type that gets called for each file or directory visited by the Walker. -type walkFunc func(path string, _ os.FileInfo, _ io.Reader) error - -// walker is the interface that walks through a file tree, -// calling walk for each file or directory in the tree. -type walker interface { - walk(walkFunc) error - reader -} - -type tarWalker struct { - r io.ReadSeeker - tarReader -} - -// newTarWalker returns a Walker that walks through .tar files. -func newTarWalker(tarFile string, r io.ReadSeeker) walker { - return &tarWalker{r, tarReader{name: tarFile}} -} - -func (w *tarWalker) walk(f walkFunc) error { - if _, err := w.r.Seek(0, os.SEEK_SET); err != nil { - return errors.Wrapf(err, "unable to reset") - } - - tr := tar.NewReader(w.r) - -loop: - for { - hdr, err := tr.Next() - switch err { - case io.EOF: - break loop - case nil: - // success, continue below - default: - return errors.Wrapf(err, "error advancing tar stream") - } - - info := hdr.FileInfo() - if err := f(hdr.Name, info, tr); err != nil { - return err - } - } - - return nil -} - -type eofReader struct{} - -func (eofReader) Read(_ []byte) (int, error) { - return 0, io.EOF -} - -type pathWalker struct { - root string - layoutReader -} - -// newPathWalker returns a Walker that walks through directories -// starting at the given root path. It does not follow symlinks. -func newPathWalker(root string) walker { - return &pathWalker{root, layoutReader{root: root}} -} - -func (w *pathWalker) walk(f walkFunc) error { - return filepath.Walk(w.root, func(path string, info os.FileInfo, err error) error { - // MUST check error value, to make sure the `os.FileInfo` is available. - // Otherwise panic risk will exist. - if err != nil { - return errors.Wrap(err, "error walking path") - } - - rel, err := filepath.Rel(w.root, path) - if err != nil { - return errors.Wrap(err, "error walking path") // err from filepath.Walk includes path name - } - - if info.IsDir() { // behave like a tar reader for directories - return f(rel, info, eofReader{}) - } - - file, err := os.Open(path) - if err != nil { - return errors.Wrap(err, "unable to open file") // os.Open includes the path - } - defer file.Close() - - return f(rel, info, file) - }) -} From 94f5137a3fd57a9f87356c69da25e408fb93dcbe Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 31 Aug 2016 00:22:19 -0700 Subject: [PATCH 12/14] cmd: Document the cas, refs, and init commands Most of this is new boilerplate, but oci-image-tools.7.md is based on the old oci-image-tool.1.md removed by fe363aa0 (*: move to opencontainers/image-tools, 2016-09-15). There's a lot going on in this repo, and it's nice to have a page that outlines everything provided by the project even though we're no longer providing a single command. Signed-off-by: W. Trevor King --- cmd/oci-cas/oci-cas-get.1.md | 27 +++++++++ cmd/oci-cas/oci-cas-put.1.md | 27 +++++++++ cmd/oci-cas/oci-cas.1.md | 47 ++++++++++++++++ .../oci-create-runtime-bundle.1.md | 2 +- .../oci-image-init-image-layout.1.md | 27 +++++++++ cmd/oci-image-init/oci-image-init.1.md | 33 +++++++++++ cmd/oci-image-tools.7.md | 40 ++++++++++++++ .../oci-image-validate.1.md | 2 +- cmd/oci-refs/oci-refs-get.1.md | 27 +++++++++ cmd/oci-refs/oci-refs-list.1.md | 27 +++++++++ cmd/oci-refs/oci-refs-put.1.md | 27 +++++++++ cmd/oci-refs/oci-refs.1.md | 55 +++++++++++++++++++ cmd/oci-unpack/oci-unpack.1.md | 2 +- 13 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 cmd/oci-cas/oci-cas-get.1.md create mode 100644 cmd/oci-cas/oci-cas-put.1.md create mode 100644 cmd/oci-cas/oci-cas.1.md create mode 100644 cmd/oci-image-init/oci-image-init-image-layout.1.md create mode 100644 cmd/oci-image-init/oci-image-init.1.md create mode 100644 cmd/oci-image-tools.7.md create mode 100644 cmd/oci-refs/oci-refs-get.1.md create mode 100644 cmd/oci-refs/oci-refs-list.1.md create mode 100644 cmd/oci-refs/oci-refs-put.1.md create mode 100644 cmd/oci-refs/oci-refs.1.md diff --git a/cmd/oci-cas/oci-cas-get.1.md b/cmd/oci-cas/oci-cas-get.1.md new file mode 100644 index 0000000..5130b29 --- /dev/null +++ b/cmd/oci-cas/oci-cas-get.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-cas-get \- Retrieve a blob from the store + +# SYNOPSIS + +**oci-cas get** [OPTIONS] PATH DIGEST + +# DESCRIPTION + +`oci-cas get` retrieves a blob referenced by `DIGEST` from the store at `PATH` and writes it to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-cas**(1), **oci-cas-put**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/oci-cas-put.1.md b/cmd/oci-cas/oci-cas-put.1.md new file mode 100644 index 0000000..56862c6 --- /dev/null +++ b/cmd/oci-cas/oci-cas-put.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-cas-put \- Write a blob to the store + +# SYNOPSIS + +**oci-cas put** [OPTIONS] PATH + +# DESCRIPTION + +`oci-cas put` reads a blob from stdin, writes it to the store at `PATH`, and prints the digest to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-cas**(1), **oci-cas-get**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/oci-cas.1.md b/cmd/oci-cas/oci-cas.1.md new file mode 100644 index 0000000..2e95ced --- /dev/null +++ b/cmd/oci-cas/oci-cas.1.md @@ -0,0 +1,47 @@ +% OCI(1) OCI-User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-cas \- Content-addressable storage manipulation + +# SYNOPSIS + +**oci-cas** [command] + +# DESCRIPTION + +`oci-cas` manipulates content-addressable storage. + +# OPTIONS + +**--help** + Print usage statement + +# COMMANDS + +**get** + Retrieve a blob from the store. + See **oci-cas-get**(1) for full documentation on the **get** command. + +**put** + Write a blob to the store. + See **oci-cas-put**(1) for full documentation on the **put** command. + +# EXAMPLES + +``` +$ oci-image-init image-layout image.tar +$ echo hello | oci-cas put image.tar +sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +$ oci-cas get image.tar sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +hello +``` + +# SEE ALSO + +**oci-image-tools**(7), **oci-cas-get**(1), **oci-cas-put**(1), **oci-image-init**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md index 8fee487..cd3223d 100644 --- a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md +++ b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md @@ -36,7 +36,7 @@ $ cd busybox-bundle && sudo runc run busybox ``` # SEE ALSO -**runc**(1), **skopeo**(1) +**oci-image-tools**(7), **runc**(1), **skopeo**(1) # HISTORY Sept 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/cmd/oci-image-init/oci-image-init-image-layout.1.md b/cmd/oci-image-init/oci-image-init-image-layout.1.md new file mode 100644 index 0000000..40a90ac --- /dev/null +++ b/cmd/oci-image-init/oci-image-init-image-layout.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-image-init-image-layout \- Initialize an OCI image-layout repository + +# SYNOPSIS + +**oci-image-init image-layout** [OPTIONS] PATH + +# DESCRIPTION + +`oci-image-init image-layout` initializes an image-layout repository at `PATH`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-image-init**(1), **oci-cas**(1), ***oci-refs**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-init/oci-image-init.1.md b/cmd/oci-image-init/oci-image-init.1.md new file mode 100644 index 0000000..3f1eaa9 --- /dev/null +++ b/cmd/oci-image-init/oci-image-init.1.md @@ -0,0 +1,33 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-image-init \- Initialize an OCI image + +# SYNOPSIS + +**oci-image-init** [command] + +# DESCRIPTION + +`oci-image-init` Initializes an OCI image. + +# OPTIONS + +**--help** + Print usage statement + +# COMMANDS + +**image-layout** + Initialize an OCI image-layout repository. + See **oci-image-init-image-layout**(1) for full documentation on the **image-layout** command. + +# SEE ALSO + +**oci-image-tool**(1), **oci-image-init-image-layout**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tools.7.md b/cmd/oci-image-tools.7.md new file mode 100644 index 0000000..f9c0754 --- /dev/null +++ b/cmd/oci-image-tools.7.md @@ -0,0 +1,40 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% JULY 2016 +# NAME + +oci-image-tools \- OCI (Open Container Initiative) image tools + +# DESCRIPTION + +The OCI image tools are a collection of tools for working with the OCI image specification. + +# COMMANDS + +**oci-cas**(1) + Content-addressable storage manipulation. + +**oci-create-runtime-bundle**(1) + Create an OCI image runtime bundle + +**oci-image-init**(1) + Initialize an OCI image. + +**oci-refs**(1) + Name-based reference manipulation. + +**oci-unpack**(1) + Unpack an image or image source layout + +**oci-validate**(1) + Validate one or more image files + +# SEE ALSO + +**skopeo**(1), +/~https://github.com/opencontainers/image-spec, +/~https://github.com/opencontainers/image-tools + +# HISTORY + +July 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/cmd/oci-image-validate/oci-image-validate.1.md b/cmd/oci-image-validate/oci-image-validate.1.md index cf04249..cec1250 100644 --- a/cmd/oci-image-validate/oci-image-validate.1.md +++ b/cmd/oci-image-validate/oci-image-validate.1.md @@ -33,7 +33,7 @@ busybox-oci: OK ``` # SEE ALSO -**skopeo**(1) +**oci-image-tools**(7), **skopeo**(1) # HISTORY Sept 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) diff --git a/cmd/oci-refs/oci-refs-get.1.md b/cmd/oci-refs/oci-refs-get.1.md new file mode 100644 index 0000000..0c2ae25 --- /dev/null +++ b/cmd/oci-refs/oci-refs-get.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs-get \- Retrieve a reference from the store + +# SYNOPSIS + +**oci-refs get** [OPTIONS] PATH NAME + +# DESCRIPTION + +`oci-refs get` retrieves reference `NAME` from the store at `PATH` and writes the JSON descriptor to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-list**(1), **oci-refs-put**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs-list.1.md b/cmd/oci-refs/oci-refs-list.1.md new file mode 100644 index 0000000..96ef9b1 --- /dev/null +++ b/cmd/oci-refs/oci-refs-list.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs-list \- Return available names from the store + +# SYNOPSIS + +**oci-refs list** [OPTIONS] PATH + +# DESCRIPTION + +`oci-refs list` retrieves all names from the store at `PATH` and writes them to standard output. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-put**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs-put.1.md b/cmd/oci-refs/oci-refs-put.1.md new file mode 100644 index 0000000..97ce05f --- /dev/null +++ b/cmd/oci-refs/oci-refs-put.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs-put \- Write a reference to the store + +# SYNOPSIS + +**oci-refs put** [OPTIONS] PATH NAME + +# DESCRIPTION + +`oci-refs put` reads descriptor JSON from standard input and writes it to the store at `PATH` as `NAME`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-list**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs.1.md b/cmd/oci-refs/oci-refs.1.md new file mode 100644 index 0000000..22b8bf1 --- /dev/null +++ b/cmd/oci-refs/oci-refs.1.md @@ -0,0 +1,55 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME + +oci-refs \- Name-based reference manipulation + +# SYNOPSIS + +**oci-refs** [command] + +# DESCRIPTION + +`oci-refs` manipulates name-based references. + +# OPTIONS + +**--help** + Print usage statement + +# COMMANDS + +**get** + Retrieve a reference from the store. + See **oci-refs-get**(1) for full documentation on the **get** command. + +**list** + Return available names from the store. + See **oci-refs-list**(1) for full documentation on the **list** command. + +**put** + Write a reference to the store. + See **oci-refs-put**(1) for full documentation on the **put** command. + +# EXAMPLES + +``` +$ oci-image-init image-layout image.tar +$ DIGEST=$(echo hello | oci-cas put image.tar) +$ SIZE=$(echo hello | wc -c) +$ printf '{"mediaType": "text/plain", "digest": "%s", "size": %d}' "${DIGEST}" "${SIZE}" | +> oci-refs put image.tar greeting +$ oci-refs list image.tar +greeting +$ oci-refs get image.tar greeting +{"mediaType":"text/plain","digest":"sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03","size":6} +``` + +# SEE ALSO + +**oci-image-tools**(7), **oci-cas-put**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-put**(1) + +# HISTORY + +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-unpack/oci-unpack.1.md b/cmd/oci-unpack/oci-unpack.1.md index 4e9956d..33f2854 100644 --- a/cmd/oci-unpack/oci-unpack.1.md +++ b/cmd/oci-unpack/oci-unpack.1.md @@ -43,7 +43,7 @@ busybox-bundle ``` # SEE ALSO -**skopeo**(1) +**oci-image-tools**(7), **skopeo**(1) # HISTORY Sept 2016, Originally compiled by Antonio Murdaca (runcom at redhat dot com) From 2564e3db6a1d2c48a87e5d842ebda6c61c6686b4 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 28 Jun 2016 22:08:44 -0700 Subject: [PATCH 13/14] .tool/lint: Ignore dupl complaints for cmd/oci-*/get.go Don't worry about: $ make lint checking lint image/cas/layout/dir.go:37::warning: duplicate of image/refs/layout/dir.go:37-54 (dupl) image/refs/layout/dir.go:37::warning: duplicate of image/cas/layout/dir.go:37-54 (dupl) cmd/oci-cas/delete.go:41::warning: duplicate of cmd/oci-cas/get.go:43-62 (dupl) cmd/oci-refs/delete.go:41::warning: duplicate of cmd/oci-refs/get.go:42-61 (dupl) cmd/oci-cas/delete.go:15::warning: duplicate of cmd/oci-refs/delete.go:15-72 (dupl) cmd/oci-refs/delete.go:15::warning: duplicate of cmd/oci-cas/delete.go:15-72 (dupl) cmd/oci-cas/get.go:43::warning: duplicate of cmd/oci-refs/get.go:42-61 (dupl) cmd/oci-refs/get.go:42::warning: duplicate of cmd/oci-cas/get.go:43-62 (dupl) make: *** [lint] Error 1 The commands are all similar (open an engine, perform some method, print the result), but are short enough that extracting out helpers would be more trouble and indirection than it's worth. Oddly, dupl seems happy to print: "duplicate of oci-cas/get.go:..." and "duplicate of get.go:..." if I exclude: "duplicate of cmd/oci-cas/get.go:..." or "duplicate of .*oci-cas/get.go:..." I want to get "oci-cas" and "oci-refs" in the exclusion regular expression somewhere to avoid accidentally skipping dupl checks for other get.go and similar if they show up somewhere else in the repository, so I'm matching on the initial filename. Signed-off-by: W. Trevor King --- .tool/lint | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.tool/lint b/.tool/lint index 4a0120e..bcbcda7 100755 --- a/.tool/lint +++ b/.tool/lint @@ -14,6 +14,12 @@ for d in $(find . -type d -not -iwholename '*.git*' -a -not -iname '.tool' -a -n --exclude='error return value not checked.*(Close|Log|Print).*\(errcheck\)$' \ --exclude='.*_test\.go:.*error return value not checked.*\(errcheck\)$' \ --exclude='duplicate of.*_test.go.*\(dupl\)$' \ + --exclude='^cmd/oci-cas/delete.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-cas/get.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-refs/delete.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-refs/get.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^image/cas/layout/dir.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^image/refs/layout/dir.go:.* duplicate of .* \(dupl\)$' \ --exclude='schema/fs.go' \ --exclude='duplicate of.*main.go.*\(dupl\)$' \ --disable=aligncheck \ From ca9028412f9923dab75bfaf14a3ca65311988804 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 16 Sep 2016 22:30:41 -0700 Subject: [PATCH 14/14] image: Add image-layout directory based CAS and ref engines These are much nicer than the tar engines (hooray atomic renames :), so switch the manifest tests tests back to using the directory-backed engines. I also switched the man-page examples over to directory-backed layouts, now that they're what oci-image-init generates by default. And I added command-line wrappers for the delete methods, now that we have a backend that implements it. I do with there was a paginated, callback-based directory lister we could use instead of ioutils.ReadDir. On the other hand, by the time directories get big enough for that to matter we may be sharding them anyway. Signed-off-by: W. Trevor King --- cmd/oci-cas/delete.go | 72 ++++++++ cmd/oci-cas/main.go | 1 + cmd/oci-cas/oci-cas-delete.1.md | 27 +++ cmd/oci-cas/oci-cas-get.1.md | 2 +- cmd/oci-cas/oci-cas-put.1.md | 2 +- cmd/oci-cas/oci-cas.1.md | 13 +- cmd/oci-image-init/image_layout.go | 29 ++- .../oci-image-init-image-layout.1.md | 5 + cmd/oci-refs/delete.go | 72 ++++++++ cmd/oci-refs/main.go | 1 + cmd/oci-refs/oci-refs-delete.1.md | 27 +++ cmd/oci-refs/oci-refs-get.1.md | 2 +- cmd/oci-refs/oci-refs-list.1.md | 2 +- cmd/oci-refs/oci-refs-put.1.md | 2 +- cmd/oci-refs/oci-refs.1.md | 17 +- image/cas/layout/dir.go | 122 +++++++++++++ image/cas/layout/main.go | 11 +- image/layout/dir.go | 98 ++++++++++ image/layout/layout.go | 28 +++ image/layout/tar.go | 12 +- image/manifest_test.go | 2 +- image/refs/layout/dir.go | 167 ++++++++++++++++++ image/refs/layout/main.go | 12 +- 23 files changed, 688 insertions(+), 38 deletions(-) create mode 100644 cmd/oci-cas/delete.go create mode 100644 cmd/oci-cas/oci-cas-delete.1.md create mode 100644 cmd/oci-refs/delete.go create mode 100644 cmd/oci-refs/oci-refs-delete.1.md create mode 100644 image/cas/layout/dir.go create mode 100644 image/layout/dir.go create mode 100644 image/refs/layout/dir.go diff --git a/cmd/oci-cas/delete.go b/cmd/oci-cas/delete.go new file mode 100644 index 0000000..f6c2c25 --- /dev/null +++ b/cmd/oci-cas/delete.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type deleteCmd struct { + path string + digest string +} + +func newDeleteCmd() *cobra.Command { + state := &deleteCmd{} + + return &cobra.Command{ + Use: "delete PATH DIGEST", + Short: "Remove a blob from from the store", + Run: state.Run, + } +} + +func (state *deleteCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and DIGEST must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.digest = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *deleteCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.Delete(ctx, state.digest) +} diff --git a/cmd/oci-cas/main.go b/cmd/oci-cas/main.go index c14f376..59bff68 100644 --- a/cmd/oci-cas/main.go +++ b/cmd/oci-cas/main.go @@ -29,6 +29,7 @@ func main() { cmd.AddCommand(newGetCmd()) cmd.AddCommand(newPutCmd()) + cmd.AddCommand(newDeleteCmd()) err := cmd.Execute() if err != nil { diff --git a/cmd/oci-cas/oci-cas-delete.1.md b/cmd/oci-cas/oci-cas-delete.1.md new file mode 100644 index 0000000..09d444f --- /dev/null +++ b/cmd/oci-cas/oci-cas-delete.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% SEPTEMBER 2016 +# NAME + +oci-cas-get \- Remove a blob from the store + +# SYNOPSIS + +**oci-cas delete** [OPTIONS] PATH DIGEST + +# DESCRIPTION + +`oci-cas delete` removes the blob referenced by `DIGEST` from the store at `PATH`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-cas**(1), **oci-cas-get**(1), **oci-cas-put**(1) + +# HISTORY + +September 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-cas/oci-cas-get.1.md b/cmd/oci-cas/oci-cas-get.1.md index 5130b29..3d55466 100644 --- a/cmd/oci-cas/oci-cas-get.1.md +++ b/cmd/oci-cas/oci-cas-get.1.md @@ -20,7 +20,7 @@ oci-cas-get \- Retrieve a blob from the store # SEE ALSO -**oci-cas**(1), **oci-cas-put**(1) +**oci-cas**(1), **oci-cas-put**(1), **oci-cas-delete**(1) # HISTORY diff --git a/cmd/oci-cas/oci-cas-put.1.md b/cmd/oci-cas/oci-cas-put.1.md index 56862c6..ffe867b 100644 --- a/cmd/oci-cas/oci-cas-put.1.md +++ b/cmd/oci-cas/oci-cas-put.1.md @@ -20,7 +20,7 @@ oci-cas-put \- Write a blob to the store # SEE ALSO -**oci-cas**(1), **oci-cas-get**(1) +**oci-cas**(1), **oci-cas-get**(1), **oci-cas-delete**(1) # HISTORY diff --git a/cmd/oci-cas/oci-cas.1.md b/cmd/oci-cas/oci-cas.1.md index 2e95ced..9c8912d 100644 --- a/cmd/oci-cas/oci-cas.1.md +++ b/cmd/oci-cas/oci-cas.1.md @@ -28,19 +28,24 @@ oci-cas \- Content-addressable storage manipulation Write a blob to the store. See **oci-cas-put**(1) for full documentation on the **put** command. +**delete** + Remove a blob from the store. + See **oci-cas-delete**(1) for full documentation on the **delete** command. + # EXAMPLES ``` -$ oci-image-init image-layout image.tar -$ echo hello | oci-cas put image.tar +$ oci-image-init image-layout image +$ echo hello | oci-cas put image sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 -$ oci-cas get image.tar sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +$ oci-cas get image sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 hello +$ oci-cas delete image sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` # SEE ALSO -**oci-image-tools**(7), **oci-cas-get**(1), **oci-cas-put**(1), **oci-image-init**(1) +**oci-image-tools**(7), **oci-cas-get**(1), **oci-cas-put**(1), **oci-cas-delete**(1), **oci-image-init**(1) # HISTORY diff --git a/cmd/oci-image-init/image_layout.go b/cmd/oci-image-init/image_layout.go index 10b8959..f245541 100644 --- a/cmd/oci-image-init/image_layout.go +++ b/cmd/oci-image-init/image_layout.go @@ -23,21 +23,33 @@ import ( "golang.org/x/net/context" ) -type imageLayout struct{} +type imageLayout struct { + backend string +} func newImageLayoutCmd() *cobra.Command { - state := &imageLayout{} + state := &imageLayout{ + backend: "dir", + } - return &cobra.Command{ + cmd := &cobra.Command{ Use: "image-layout PATH", Short: "Initialize an OCI image-layout repository", Run: state.Run, } + + cmd.Flags().StringVar( + &state.backend, "type", "dir", + "Select the image-backend. Choices: dir, tar. Defaults to dir.", + ) + + return cmd } func (state *imageLayout) Run(cmd *cobra.Command, args []string) { + var err error if len(args) != 1 { - if err := cmd.Usage(); err != nil { + if err = cmd.Usage(); err != nil { fmt.Fprintln(os.Stderr, err) } os.Exit(1) @@ -47,7 +59,14 @@ func (state *imageLayout) Run(cmd *cobra.Command, args []string) { ctx := context.Background() - err := layout.CreateTarFile(ctx, path) + switch state.backend { + case "dir": + err = layout.CreateDir(ctx, path) + case "tar": + err = layout.CreateTarFile(ctx, path) + default: + err = fmt.Errorf("unrecognized type: %q", state.backend) + } if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/cmd/oci-image-init/oci-image-init-image-layout.1.md b/cmd/oci-image-init/oci-image-init-image-layout.1.md index 40a90ac..3351bbf 100644 --- a/cmd/oci-image-init/oci-image-init-image-layout.1.md +++ b/cmd/oci-image-init/oci-image-init-image-layout.1.md @@ -18,6 +18,11 @@ oci-image-init-image-layout \- Initialize an OCI image-layout repository **--help** Print usage statement +**--type** + Select the image-layout backend. + Choices: dir, tar. + Defaults to dir. + # SEE ALSO **oci-image-init**(1), **oci-cas**(1), ***oci-refs**(1) diff --git a/cmd/oci-refs/delete.go b/cmd/oci-refs/delete.go new file mode 100644 index 0000000..0e661cd --- /dev/null +++ b/cmd/oci-refs/delete.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/opencontainers/image-tools/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type deleteCmd struct { + path string + name string +} + +func newDeleteCmd() *cobra.Command { + state := &deleteCmd{} + + return &cobra.Command{ + Use: "delete PATH NAME", + Short: "Remove a reference from the store", + Run: state.Run, + } +} + +func (state *deleteCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + fmt.Fprintln(os.Stderr, "both PATH and NAME must be provided") + if err := cmd.Usage(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *deleteCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.Delete(ctx, state.name) +} diff --git a/cmd/oci-refs/main.go b/cmd/oci-refs/main.go index 35ea2c1..e7a96cb 100644 --- a/cmd/oci-refs/main.go +++ b/cmd/oci-refs/main.go @@ -30,6 +30,7 @@ func main() { cmd.AddCommand(newPutCmd()) cmd.AddCommand(newGetCmd()) cmd.AddCommand(newListCmd()) + cmd.AddCommand(newDeleteCmd()) err := cmd.Execute() if err != nil { diff --git a/cmd/oci-refs/oci-refs-delete.1.md b/cmd/oci-refs/oci-refs-delete.1.md new file mode 100644 index 0000000..b477743 --- /dev/null +++ b/cmd/oci-refs/oci-refs-delete.1.md @@ -0,0 +1,27 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% SEPTEMBER 2016 +# NAME + +oci-refs-delete \- Remove a reference from the store + +# SYNOPSIS + +**oci-refs delete** [OPTIONS] PATH NAME + +# DESCRIPTION + +`oci-refs delete` removes reference `NAME` from the store at `PATH`. + +# OPTIONS + +**--help** + Print usage statement + +# SEE ALSO + +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-put**(1) + +# HISTORY + +September 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-refs/oci-refs-get.1.md b/cmd/oci-refs/oci-refs-get.1.md index 0c2ae25..49afd8d 100644 --- a/cmd/oci-refs/oci-refs-get.1.md +++ b/cmd/oci-refs/oci-refs-get.1.md @@ -20,7 +20,7 @@ oci-refs-get \- Retrieve a reference from the store # SEE ALSO -**oci-refs**(1), **oci-refs-list**(1), **oci-refs-put**(1) +**oci-refs**(1), **oci-refs-list**(1), **oci-refs-put**(1), **oci-refs-delete**(1) # HISTORY diff --git a/cmd/oci-refs/oci-refs-list.1.md b/cmd/oci-refs/oci-refs-list.1.md index 96ef9b1..38c610b 100644 --- a/cmd/oci-refs/oci-refs-list.1.md +++ b/cmd/oci-refs/oci-refs-list.1.md @@ -20,7 +20,7 @@ oci-refs-list \- Return available names from the store # SEE ALSO -**oci-refs**(1), **oci-refs-get**(1), **oci-refs-put**(1) +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-put**(1), **oci-refs-delete**(1) # HISTORY diff --git a/cmd/oci-refs/oci-refs-put.1.md b/cmd/oci-refs/oci-refs-put.1.md index 97ce05f..61c5098 100644 --- a/cmd/oci-refs/oci-refs-put.1.md +++ b/cmd/oci-refs/oci-refs-put.1.md @@ -20,7 +20,7 @@ oci-refs-put \- Write a reference to the store # SEE ALSO -**oci-refs**(1), **oci-refs-get**(1), **oci-refs-list**(1) +**oci-refs**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-delete**(1) # HISTORY diff --git a/cmd/oci-refs/oci-refs.1.md b/cmd/oci-refs/oci-refs.1.md index 22b8bf1..5f9a284 100644 --- a/cmd/oci-refs/oci-refs.1.md +++ b/cmd/oci-refs/oci-refs.1.md @@ -32,23 +32,28 @@ oci-refs \- Name-based reference manipulation Write a reference to the store. See **oci-refs-put**(1) for full documentation on the **put** command. +**delete** + Remove a reference from the store. + See **oci-refs-delete**(1) for full documentation on the **delete** command. + # EXAMPLES ``` -$ oci-image-init image-layout image.tar -$ DIGEST=$(echo hello | oci-cas put image.tar) +$ oci-image-init image-layout image +$ DIGEST=$(echo hello | oci-cas put image) $ SIZE=$(echo hello | wc -c) $ printf '{"mediaType": "text/plain", "digest": "%s", "size": %d}' "${DIGEST}" "${SIZE}" | -> oci-refs put image.tar greeting -$ oci-refs list image.tar +> oci-refs put image greeting +$ oci-refs list image greeting -$ oci-refs get image.tar greeting +$ oci-refs get image greeting {"mediaType":"text/plain","digest":"sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03","size":6} +$ oci-refs delete image greeting ``` # SEE ALSO -**oci-image-tools**(7), **oci-cas-put**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-put**(1) +**oci-image-tools**(7), **oci-cas-put**(1), **oci-refs-get**(1), **oci-refs-list**(1), **oci-refs-put**(1), **oci-refs-delete**(1) # HISTORY diff --git a/image/cas/layout/dir.go b/image/cas/layout/dir.go new file mode 100644 index 0000000..7e729c6 --- /dev/null +++ b/image/cas/layout/dir.go @@ -0,0 +1,122 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "crypto/sha256" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/opencontainers/image-tools/image/cas" + "github.com/opencontainers/image-tools/image/layout" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// DirEngine is a cas.Engine backed by a directory. +type DirEngine struct { + path string + temp string +} + +// NewDirEngine returns a new DirEngine. +func NewDirEngine(ctx context.Context, path string) (eng cas.Engine, err error) { + engine := &DirEngine{ + path: path, + } + + err = layout.CheckDirVersion(ctx, engine.path) + if err != nil { + return nil, err + } + + tempDir, err := ioutil.TempDir(path, "tmp-") + if err != nil { + return nil, err + } + engine.temp = tempDir + + return engine, nil +} + +// Put adds a new blob to the store. +func (engine *DirEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { + hash := sha256.New() + algorithm := "sha256" + + var file *os.File + file, err = ioutil.TempFile(engine.temp, "blob-") + if err != nil { + return "", err + } + defer func() { + if err != nil { + err2 := os.Remove(file.Name()) + if err2 != nil { + err = errors.Wrap(err, err2.Error()) + } + } + }() + defer file.Close() + + hashingWriter := io.MultiWriter(file, hash) + _, err = io.Copy(hashingWriter, reader) + if err != nil { + return "", err + } + file.Close() + + digest = fmt.Sprintf("%s:%x", algorithm, hash.Sum(nil)) + targetName, err := blobPath(digest, string(os.PathSeparator)) + if err != nil { + return "", err + } + + path := filepath.Join(engine.path, targetName) + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + return "", err + } + + err = os.Rename(file.Name(), path) + if err != nil { + return "", err + } + + return digest, nil +} + +// Get returns a reader for retrieving a blob from the store. +func (engine *DirEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) { + targetName, err := blobPath(digest, string(os.PathSeparator)) + if err != nil { + return nil, err + } + + return os.Open(filepath.Join(engine.path, targetName)) +} + +// Delete removes a blob from the store. +func (engine *DirEngine) Delete(ctx context.Context, digest string) (err error) { + return layout.DirDelete(ctx, engine.path, digest, blobPath) +} + +// Close releases resources held by the engine. +func (engine *DirEngine) Close() (err error) { + return os.RemoveAll(engine.temp) +} diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go index d9972a6..de9ab9e 100644 --- a/image/cas/layout/main.go +++ b/image/cas/layout/main.go @@ -30,12 +30,17 @@ import ( // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) { + engine, err = NewDirEngine(ctx, path) + if err == nil { + return engine, err + } + file, err := os.OpenFile(path, os.O_RDWR, 0) - if err != nil { - return nil, err + if err == nil { + return NewTarEngine(ctx, file) } - return NewTarEngine(ctx, file) + return nil, fmt.Errorf("unrecognized engine at %q", path) } // blobPath returns the PATH to the DIGEST blob. SEPARATOR selects diff --git a/image/layout/dir.go b/image/layout/dir.go new file mode 100644 index 0000000..a46fb54 --- /dev/null +++ b/image/layout/dir.go @@ -0,0 +1,98 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/net/context" +) + +// CheckDirVersion checks the oci-layout entry in an image-layout +// directory and returns an error if oci-layout is missing or has +// unrecognized content. +func CheckDirVersion(ctx context.Context, path string) (err error) { + file, err := os.Open(filepath.Join(path, "oci-layout")) + if os.IsNotExist(err) { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + defer file.Close() + + return CheckVersion(ctx, file) +} + +// CreateDir creates a new image-layout directory at the given path. +func CreateDir(ctx context.Context, path string) (err error) { + err = os.MkdirAll(path, 0777) + if err != nil { + return err + } + + file, err := os.OpenFile( + filepath.Join(path, "oci-layout"), + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + 0666, + ) + if err != nil { + return err + } + defer file.Close() + + imageLayoutVersion := ImageLayoutVersion{ + Version: "1.0.0", + } + imageLayoutVersionBytes, err := json.Marshal(imageLayoutVersion) + if err != nil { + return err + } + n, err := file.Write(imageLayoutVersionBytes) + if err != nil { + return err + } + if n < len(imageLayoutVersionBytes) { + return fmt.Errorf("wrote %d of %d bytes", n, len(imageLayoutVersionBytes)) + } + + err = os.MkdirAll(filepath.Join(path, "blobs"), 0777) + if err != nil { + return err + } + + return os.MkdirAll(filepath.Join(path, "refs"), 0777) +} + +// DirDelete removes an entry from a directory, wrapping a +// path-constructing call to entryPath and a removing call to +// os.Remove. Deletion is idempotent (unlike os.Remove, where +// attempting to delete a nonexistent path results in an error). +func DirDelete(ctx context.Context, path string, entry string, entryPath EntryPath) (err error) { + targetName, err := entryPath(entry, string(os.PathSeparator)) + if err != nil { + return err + } + + err = os.Remove(filepath.Join(path, targetName)) + if os.IsNotExist(err) { + return nil + } + return err +} diff --git a/image/layout/layout.go b/image/layout/layout.go index e6989fe..cf1ba7b 100644 --- a/image/layout/layout.go +++ b/image/layout/layout.go @@ -15,8 +15,36 @@ // Package layout defines utility code shared by refs/layout and cas/layout. package layout +import ( + "encoding/json" + "fmt" + "io" + + "golang.org/x/net/context" +) + +// EntryPath is a template for helpers that convert from ref or blob +// names to image-layout paths. +type EntryPath func(entry string, separator string) (path string, err error) + // ImageLayoutVersion represents the oci-version content for the image // layout format. type ImageLayoutVersion struct { Version string `json:"imageLayoutVersion"` } + +// CheckVersion checks an oci-layout reader and returns an error if it +// has unrecognized content. +func CheckVersion(ctx context.Context, reader io.Reader) (err error) { + decoder := json.NewDecoder(reader) + var version ImageLayoutVersion + err = decoder.Decode(&version) + if err != nil { + return err + } + if version.Version != "1.0.0" { + return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) + } + + return nil +} diff --git a/image/layout/tar.go b/image/layout/tar.go index 0b2ad85..81f5a7f 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -206,17 +206,7 @@ func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { return err } - decoder := json.NewDecoder(tarReader) - var version ImageLayoutVersion - err = decoder.Decode(&version) - if err != nil { - return err - } - if version.Version != "1.0.0" { - return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) - } - - return nil + return CheckVersion(ctx, tarReader) } // CreateTarFile creates a new image-layout tar file at the given path. diff --git a/image/manifest_test.go b/image/manifest_test.go index 96b1cdb..64c98e4 100644 --- a/image/manifest_test.go +++ b/image/manifest_test.go @@ -79,7 +79,7 @@ func TestUnpackLayer(t *testing.T) { defer os.RemoveAll(tmp1) path := filepath.Join(tmp1, "image.tar") - err = imagelayout.CreateTarFile(ctx, path) + err = imagelayout.CreateDir(ctx, path) if err != nil { t.Fatal(err) } diff --git a/image/refs/layout/dir.go b/image/refs/layout/dir.go new file mode 100644 index 0000000..af33fe4 --- /dev/null +++ b/image/refs/layout/dir.go @@ -0,0 +1,167 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/layout" + "github.com/opencontainers/image-tools/image/refs" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// DirEngine is a refs.Engine backed by a directory. +type DirEngine struct { + path string + temp string +} + +// NewDirEngine returns a new DirEngine. +func NewDirEngine(ctx context.Context, path string) (eng refs.Engine, err error) { + engine := &DirEngine{ + path: path, + } + + err = layout.CheckDirVersion(ctx, engine.path) + if err != nil { + return nil, err + } + + tempDir, err := ioutil.TempDir(path, "tmp-") + if err != nil { + return nil, err + } + engine.temp = tempDir + + return engine, nil +} + +// Put adds a new reference to the store. +func (engine *DirEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { + var file *os.File + file, err = ioutil.TempFile(engine.temp, "ref-") + if err != nil { + return err + } + defer func() { + if err != nil { + err2 := os.Remove(file.Name()) + if err2 != nil { + err = errors.Wrap(err, err2.Error()) + } + } + }() + defer file.Close() + + encoder := json.NewEncoder(file) + err = encoder.Encode(descriptor) + if err != nil { + return err + } + + err = file.Close() + if err != nil { + return err + } + + targetName, err := refPath(name, string(os.PathSeparator)) + if err != nil { + return err + } + + path := filepath.Join(engine.path, targetName) + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + return err + } + + return os.Rename(file.Name(), path) +} + +// Get returns a reference from the store. +func (engine *DirEngine) Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) { + targetName, err := refPath(name, string(os.PathSeparator)) + if err != nil { + return nil, err + } + + var file *os.File + file, err = os.Open(filepath.Join(engine.path, targetName)) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(file) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err + } + return &desc, nil +} + +// List returns available names from the store. +func (engine *DirEngine) List(ctx context.Context, prefix string, size int, from int, nameCallback refs.ListNameCallback) (err error) { + var i = 0 + + pathPrefix, err := refPath(prefix, string(os.PathSeparator)) + if err != nil { + return nil + } + var files []os.FileInfo + files, err = ioutil.ReadDir(filepath.Join(engine.path, filepath.Dir(pathPrefix))) + if err != nil { + return err + } + for _, file := range files { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + name := file.Name() + if strings.HasPrefix(name, prefix) { + i++ + if i > from { + err = nameCallback(ctx, name) + if err != nil { + return err + } + if i-from == size { + return nil + } + } + } + } + + return nil +} + +// Delete removes a reference from the store. +func (engine *DirEngine) Delete(ctx context.Context, name string) (err error) { + return layout.DirDelete(ctx, engine.path, name, refPath) +} + +// Close releases resources held by the engine. +func (engine *DirEngine) Close() (err error) { + return os.RemoveAll(engine.temp) +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go index d1722e5..41c2fb5 100644 --- a/image/refs/layout/main.go +++ b/image/refs/layout/main.go @@ -19,6 +19,7 @@ package layout import ( + "fmt" "os" "strings" @@ -29,12 +30,17 @@ import ( // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). func NewEngine(ctx context.Context, path string) (engine refs.Engine, err error) { + engine, err = NewDirEngine(ctx, path) + if err == nil { + return engine, err + } + file, err := os.OpenFile(path, os.O_RDWR, 0) - if err != nil { - return nil, err + if err == nil { + return NewTarEngine(ctx, file) } - return NewTarEngine(ctx, file) + return nil, fmt.Errorf("unrecognized engine at %q", path) } // refPath returns the PATH to the NAME reference. SEPARATOR selects