Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

llbsolver: move history blobs to a separate namespace #3833

Merged
merged 2 commits into from
May 24, 2023
Merged

llbsolver: move history blobs to a separate namespace #3833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f044e0a
llbsolver: move history blobs to a separate namespace
tonistiigi Apr 29, 2023
f5ca0c5
remove type aliases for leasemanager/contentstore
tonistiigi May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions control/control.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

contentapi "github.com/containerd/containerd/api/services/content/v1"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/services/content/contentserver"
"github.com/docker/distribution/reference"
"github.com/mitchellh/hashstructure/v2"
Expand All @@ -26,12 +25,14 @@ import (
"github.com/moby/buildkit/frontend/attestations"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/grpchijack"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/llbsolver"
"github.com/moby/buildkit/solver/llbsolver/proc"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/throttle"
"github.com/moby/buildkit/util/tracing/transform"
"github.com/moby/buildkit/version"
Expand All @@ -58,8 +59,8 @@ type Opt struct {
Entitlements []string
TraceCollector sdktrace.SpanExporter
HistoryDB *bbolt.DB
LeaseManager leases.Manager
ContentStore content.Store
LeaseManager *leaseutil.Manager
ContentStore *containerdsnapshot.Store
HistoryConfig *config.HistoryConfig
}

Expand All @@ -79,12 +80,15 @@ type Controller struct { // TODO: ControlService
func NewController(opt Opt) (*Controller, error) {
gatewayForwarder := controlgateway.NewGatewayForwarder()

hq := llbsolver.NewHistoryQueue(llbsolver.HistoryQueueOpt{
hq, err := llbsolver.NewHistoryQueue(llbsolver.HistoryQueueOpt{
DB: opt.HistoryDB,
LeaseManager: opt.LeaseManager,
ContentStore: opt.ContentStore,
CleanConfig: opt.HistoryConfig,
})
if err != nil {
return nil, errors.Wrap(err, "failed to create history queue")
}

s, err := llbsolver.New(llbsolver.Opt{
WorkerController: opt.WorkerController,
Expand Down Expand Up @@ -125,7 +129,7 @@ func (c *Controller) Register(server *grpc.Server) {
c.gatewayForwarder.Register(server)
tracev1.RegisterTraceServiceServer(server, c)

store := &roContentStore{c.opt.ContentStore}
store := &roContentStore{c.opt.ContentStore.WithFallbackNS(c.opt.ContentStore.Namespace() + "_history")}
contentapi.RegisterContentServer(server, contentserver.New(store))
}

Expand Down
97 changes: 84 additions & 13 deletions snapshot/containerd/content.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,64 +5,80 @@ import (

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/nydus-snapshotter/pkg/errdefs"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

func NewContentStore(store content.Store, ns string) content.Store {
return &nsContent{ns, store}
func NewContentStore(store content.Store, ns string) *Store {
return &Store{ns, store}
}

type nsContent struct {
type Store struct {
ns string
content.Store
}

func (c *nsContent) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
func (c *Store) Namespace() string {
return c.ns
}

func (c *Store) WithNamespace(ns string) *Store {
return NewContentStore(c.Store, ns)
}

func (c *Store) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.Info(ctx, dgst)
}

func (c *nsContent) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
func (c *Store) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.Update(ctx, info, fieldpaths...)
}

func (c *nsContent) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error {
func (c *Store) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.Walk(ctx, fn, filters...)
}

func (c *nsContent) Delete(ctx context.Context, dgst digest.Digest) error {
func (c *Store) Delete(ctx context.Context, dgst digest.Digest) error {
return errors.Errorf("contentstore.Delete usage is forbidden")
}

func (c *nsContent) Status(ctx context.Context, ref string) (content.Status, error) {
func (c *Store) Status(ctx context.Context, ref string) (content.Status, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.Status(ctx, ref)
}

func (c *nsContent) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) {
func (c *Store) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.ListStatuses(ctx, filters...)
}

func (c *nsContent) Abort(ctx context.Context, ref string) error {
func (c *Store) Abort(ctx context.Context, ref string) error {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.Abort(ctx, ref)
}

func (c *nsContent) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
func (c *Store) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.ReaderAt(ctx, desc)
}

func (c *nsContent) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
func (c *Store) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
return c.writer(ctx, 3, opts...)
}

func (c *nsContent) writer(ctx context.Context, retries int, opts ...content.WriterOpt) (content.Writer, error) {
func (c *Store) WithFallbackNS(ns string) content.Store {
return &nsFallbackStore{
main: c,
fb: c.WithNamespace(ns),
}
}

func (c *Store) writer(ctx context.Context, retries int, opts ...content.WriterOpt) (content.Writer, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
w, err := c.Store.Writer(ctx, opts...)
if err != nil {
Expand All @@ -80,3 +96,58 @@ func (w *nsWriter) Commit(ctx context.Context, size int64, expected digest.Diges
ctx = namespaces.WithNamespace(ctx, w.ns)
return w.Writer.Commit(ctx, size, expected, opts...)
}

type nsFallbackStore struct {
main *Store
fb *Store
}
Comment on lines +100 to +103
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, could we potentially use an implementation that was more similar to the content store multiplexing we already have in buildkit (added in #615)?

On the client:

buildkit/session/content/attachable.go

Lines 21 to 41 in 8dd5f87

type attachableContentStore struct {
stores map[string]content.Store
}
func (cs *attachableContentStore) choose(ctx context.Context) (content.Store, error) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return nil, errors.Wrap(errdefs.ErrInvalidArgument, "request lacks metadata")
}
values := md[GRPCHeaderID]
if len(values) == 0 {
return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "request lacks metadata %q", GRPCHeaderID)
}
id := values[0]
store, ok := cs.stores[id]
if !ok {
return nil, errors.Wrapf(errdefs.ErrNotFound, "unknown store %s", id)
}
return store, nil
}

On the server:

buildkit/session/content/caller.go

Lines 16 to 31 in 8dd5f87

type callerContentStore struct {
store content.Store
storeID string
}
func (cs *callerContentStore) choose(ctx context.Context) context.Context {
nsheader := metadata.Pairs(GRPCHeaderID, cs.storeID)
md, ok := metadata.FromOutgoingContext(ctx) // merge with outgoing context.
if !ok {
md = nsheader
} else {
// order ensures the latest is first in this list.
md = metadata.Join(nsheader, md)
}
return metadata.NewOutgoingContext(ctx, md)
}

I think we need to still need to have the fallback store as the default, to emulate v0.11 buildkit behavior, but I think maybe moving forwards the client should be able to dictate which store to use?


var _ content.Store = &nsFallbackStore{}

func (c *nsFallbackStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
info, err := c.main.Info(ctx, dgst)
if err != nil {
if errdefs.IsNotFound(err) {
return c.fb.Info(ctx, dgst)
}
}
return info, err
}

func (c *nsFallbackStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
return c.main.Update(ctx, info, fieldpaths...)
}

func (c *nsFallbackStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error {
return c.main.Walk(ctx, fn, filters...)
}

func (c *nsFallbackStore) Delete(ctx context.Context, dgst digest.Digest) error {
return c.main.Delete(ctx, dgst)
}

func (c *nsFallbackStore) Status(ctx context.Context, ref string) (content.Status, error) {
return c.main.Status(ctx, ref)
}

func (c *nsFallbackStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) {
return c.main.ListStatuses(ctx, filters...)
}

func (c *nsFallbackStore) Abort(ctx context.Context, ref string) error {
return c.main.Abort(ctx, ref)
}

func (c *nsFallbackStore) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
ra, err := c.main.ReaderAt(ctx, desc)
if err != nil {
if errdefs.IsNotFound(err) {
return c.fb.ReaderAt(ctx, desc)
}
}
return ra, err
}

func (c *nsFallbackStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
return c.main.Writer(ctx, opts...)
}
Loading