From 2423f4366cfecfe07fc0a35318985fce40b741ba Mon Sep 17 00:00:00 2001 From: Yan Song Date: Fri, 3 Mar 2023 11:55:14 +0000 Subject: [PATCH 1/3] builder: support `--parent-bootstrap` for merge This option allows merging multiple bootstraps of upper layer with the bootstrap of a parent image, so that we can implement container commit operation for nydus image. Signed-off-by: Yan Song --- src/bin/nydus-image/main.rs | 11 ++++++++- src/bin/nydus-image/merge.rs | 48 ++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 95d800bd715..d9d65534917 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -319,11 +319,17 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .subcommand( App::new("merge") .about("Merge multiple bootstraps into a overlaid bootstrap") + .arg( + Arg::new("parent-bootstrap") + .long("parent-bootstrap") + .help("File path of the parent/referenced RAFS metadata blob (optional)") + .required(false), + ) .arg( Arg::new("bootstrap") .long("bootstrap") .short('B') - .help("output path of nydus overlaid bootstrap"), + .help("Output path of nydus overlaid bootstrap"), ) .arg( Arg::new("blob-dir") @@ -921,8 +927,11 @@ impl Command { }; ctx.configuration = config.clone(); + let parent_bootstrap_path = Self::get_parent_bootstrap(matches)?; + let output = Merger::merge( &mut ctx, + parent_bootstrap_path, source_bootstrap_paths, blob_digests, blob_sizes, diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index e9aa154c60b..96def7d27c5 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashMap; use std::collections::HashSet; use std::convert::TryFrom; use std::path::{Path, PathBuf}; @@ -56,6 +57,7 @@ impl Merger { #[allow(clippy::too_many_arguments)] pub fn merge( ctx: &mut BuildContext, + parent_bootstrap_path: Option, sources: Vec, blob_digests: Option>, blob_sizes: Option>, @@ -101,6 +103,26 @@ impl Merger { ); } + let mut tree: Option = None; + let mut blob_mgr = BlobManager::new(ctx.digester); + + // Load parent bootstrap + let mut blob_idx_map = HashMap::new(); + let mut parent_layers = 0; + if let Some(parent_bootstrap_path) = &parent_bootstrap_path { + let (rs, _) = + RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false) + .context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?; + tree = Some(Tree::from_bootstrap(&rs, &mut ())?); + let blobs = rs.superblock.get_blob_infos(); + for blob in &blobs { + let blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?; + blob_idx_map.insert(blob_ctx.blob_id.clone(), blob_mgr.len()); + blob_mgr.add_blob(blob_ctx); + } + parent_layers = blobs.len(); + } + // Get the blobs come from chunk dict bootstrap. let mut chunk_dict_blobs = HashSet::new(); let mut config = None; @@ -116,8 +138,6 @@ impl Merger { let mut fs_version = RafsVersion::V6; let mut chunk_size = None; - let mut tree: Option = None; - let mut blob_mgr = BlobManager::new(ctx.digester); for (layer_idx, bootstrap_path) in sources.iter().enumerate() { let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), true, false) @@ -131,9 +151,9 @@ impl Merger { ctx.digester = rs.meta.get_digester(); ctx.explicit_uidgid = rs.meta.explicit_uidgid(); - let mut blob_idx_map = Vec::new(); let mut parent_blob_added = false; - for blob in rs.superblock.get_blob_infos() { + let blobs = &rs.superblock.get_blob_infos(); + for blob in blobs { let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?; if let Some(chunk_size) = chunk_size { ensure!( @@ -186,15 +206,8 @@ impl Merger { } } - let mut found = false; - for (idx, blob) in blob_mgr.get_blobs().iter().enumerate() { - if blob.blob_id == blob_ctx.blob_id { - blob_idx_map.push(idx as u32); - found = true; - } - } - if !found { - blob_idx_map.push(blob_mgr.len() as u32); + if !blob_idx_map.contains_key(&blob.blob_id()) { + blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len()); blob_mgr.add_blob(blob_ctx); } } @@ -213,8 +226,11 @@ impl Merger { ))?; for chunk in &mut node.chunks { let origin_blob_index = chunk.inner.blob_index() as usize; - // Set the blob index of chunk to real index in blob table of final bootstrap. - chunk.set_blob_index(blob_idx_map[origin_blob_index]); + let blob_ctx = blobs[origin_blob_index].as_ref(); + if let Some(blob_index) = blob_idx_map.get(&blob_ctx.blob_id()) { + // Set the blob index of chunk to real index in blob table of final bootstrap. + chunk.set_blob_index(*blob_index as u32); + } } // Set node's layer index to distinguish same inode number (from bootstrap) // between different layers. @@ -222,7 +238,7 @@ impl Merger { "too many layers {}, limited to {}", layer_idx, u16::MAX - ))?; + ))? + parent_layers as u16; node.overlay = Overlay::UpperAddition; match node.whiteout_type(WhiteoutSpec::Oci) { // Insert whiteouts at the head, so they will be handled first when From bee62d6a9fddedf13e8b1d1758864913f7ec9f3d Mon Sep 17 00:00:00 2001 From: Yan Song Date: Fri, 3 Mar 2023 11:55:39 +0000 Subject: [PATCH 2/3] smoke: add `--parent-bootstrap` for merge test Signed-off-by: Yan Song --- smoke/go.mod | 10 +++--- smoke/go.sum | 23 ++++++------- smoke/tests/native_layer_test.go | 55 +++++++++++++++++++++++++++++++- smoke/tests/texture/layer.go | 12 +++++++ smoke/tests/tool/layer.go | 4 ++- 5 files changed, 87 insertions(+), 17 deletions(-) diff --git a/smoke/go.mod b/smoke/go.mod index 503f4025235..feb7d399894 100644 --- a/smoke/go.mod +++ b/smoke/go.mod @@ -5,12 +5,12 @@ go 1.18 require ( github.com/containerd/containerd v1.6.18 github.com/containerd/nydus-snapshotter v0.6.1 - github.com/google/uuid v1.2.0 + github.com/google/uuid v1.3.0 github.com/opencontainers/go-digest v1.0.0 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.9 github.com/stretchr/testify v1.8.1 - golang.org/x/sys v0.4.0 + golang.org/x/sys v0.5.0 ) require ( @@ -29,9 +29,11 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + golang.org/x/sync v0.1.0 // indirect + google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect google.golang.org/grpc v1.50.1 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/containerd/nydus-snapshotter => github.com/imeoer/nydus-snapshotter v0.3.35 diff --git a/smoke/go.sum b/smoke/go.sum index a5ba8683b72..21b0356bedb 100644 --- a/smoke/go.sum +++ b/smoke/go.sum @@ -180,8 +180,6 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nydus-snapshotter v0.6.1 h1:G7k7EwnjFa1fUC3ywkldDt2BC3mtBPrt+omFke/Vdhk= -github.com/containerd/nydus-snapshotter v0.6.1/go.mod h1:U9m10GYZKisnSKOdgIjfkU8Ad0UTSYJ6CpP3I0SJBD0= github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= @@ -371,8 +369,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= @@ -404,6 +403,8 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imeoer/nydus-snapshotter v0.3.35 h1:pOW0rQrji9TpO4wsN9klpR+LA3OoNzTenbY0mUkCrHI= +github.com/imeoer/nydus-snapshotter v0.3.35/go.mod h1:haLQbzvyeVZZ59xi/DqW4uesQCAHlRW9U+GGWpiSB4s= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -749,7 +750,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -766,8 +767,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -838,8 +839,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -849,7 +850,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -948,8 +949,8 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/smoke/tests/native_layer_test.go b/smoke/tests/native_layer_test.go index 5808a9b4605..4d7f14dbed0 100644 --- a/smoke/tests/native_layer_test.go +++ b/smoke/tests/native_layer_test.go @@ -78,7 +78,6 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator { } func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) { - packOption := converter.PackOption{ BuilderPath: ctx.Binary.Builder, Compressor: ctx.Build.Compressor, @@ -154,6 +153,60 @@ func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) { lowerLayer.Overlay(t, upperLayer) ctx.Env.BootstrapPath = overlayBootstrap tool.Verify(t, ctx, lowerLayer.FileTree) + + // Make base layers (use as a parent bootstrap) + packOption.ChunkDictPath = "" + baseLayer1 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-1"), "1") + baseLayer1BlobDigest := baseLayer1.Pack(t, packOption, ctx.Env.BlobDir) + + baseLayer2 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-2"), "2") + baseLayer2BlobDigest := baseLayer2.Pack(t, packOption, ctx.Env.BlobDir) + + lowerLayer = texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "source-lower-1")) + lowerBlobDigest = lowerLayer.Pack(t, packOption, ctx.Env.BlobDir) + + upperLayer = texture.MakeUpperLayer(t, filepath.Join(ctx.Env.WorkDir, "source-upper-1")) + upperBlobDigest = upperLayer.Pack(t, packOption, ctx.Env.BlobDir) + + mergeOption = converter.MergeOption{ + BuilderPath: ctx.Binary.Builder, + } + baseLayerDigests, baseBootstrap := tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{ + { + Digest: baseLayer1BlobDigest, + }, + { + Digest: baseLayer2BlobDigest, + }, + }) + ctx.Env.BootstrapPath = baseBootstrap + require.Equal(t, []digest.Digest{baseLayer1BlobDigest, baseLayer2BlobDigest}, baseLayerDigests) + + // Test merge from a parent bootstrap + mergeOption = converter.MergeOption{ + ParentBootstrapPath: baseBootstrap, + ChunkDictPath: baseBootstrap, + BuilderPath: ctx.Binary.Builder, + } + actualDigests, overlayBootstrap = tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{ + { + Digest: lowerBlobDigest, + }, + { + Digest: upperBlobDigest, + }, + }) + + require.Equal(t, []digest.Digest{ + baseLayer1BlobDigest, + baseLayer2BlobDigest, + lowerBlobDigest, + upperBlobDigest, + }, actualDigests) + + ctx.Env.BootstrapPath = overlayBootstrap + baseLayer1.Overlay(t, baseLayer2).Overlay(t, lowerLayer).Overlay(t, upperLayer) + tool.Verify(t, ctx, baseLayer1.FileTree) } func TestNativeLayer(t *testing.T) { diff --git a/smoke/tests/texture/layer.go b/smoke/tests/texture/layer.go index 2fccde693d2..ed907904dbe 100644 --- a/smoke/tests/texture/layer.go +++ b/smoke/tests/texture/layer.go @@ -74,3 +74,15 @@ func MakeUpperLayer(t *testing.T, workDir string) *tool.Layer { return layer } + +func MakeMatrixLayer(t *testing.T, workDir, id string) *tool.Layer { + layer := tool.NewLayer(t, workDir) + + // Create regular file + file1 := fmt.Sprintf("matrix-file-%s-1", id) + file2 := fmt.Sprintf("matrix-file-%s-2", id) + layer.CreateFile(t, file1, []byte(file1)) + layer.CreateFile(t, file2, []byte(file2)) + + return layer +} diff --git a/smoke/tests/tool/layer.go b/smoke/tests/tool/layer.go index 87d9d7710b6..444808afb5a 100644 --- a/smoke/tests/tool/layer.go +++ b/smoke/tests/tool/layer.go @@ -164,7 +164,7 @@ func (l *Layer) PackRef(t *testing.T, ctx Context, blobDir string, compress bool return ociBlobDigest, rafsBlobDigest } -func (l *Layer) Overlay(t *testing.T, upper *Layer) { +func (l *Layer) Overlay(t *testing.T, upper *Layer) *Layer { // Handle whiteout/opaque files for upperName := range upper.FileTree { name := filepath.Base(upperName) @@ -198,6 +198,8 @@ func (l *Layer) Overlay(t *testing.T, upper *Layer) { } } } + + return l } func (l *Layer) recordFileTree(t *testing.T) { From a99a41fcdb05a9cc1ffb991abcade8d52211f3d8 Mon Sep 17 00:00:00 2001 From: Yan Song Date: Wed, 8 Mar 2023 03:05:13 +0000 Subject: [PATCH 3/3] rafs: do not fix blob id for old bootstrap In fact, there is no way to tell if a separate old bootstrap file was inline to the blob, for example, for an old merged bootstrap, we can't set the blob id it references to as the filename, otherwise it will break blob table on loading rafs. Signed-off-by: Yan Song --- rafs/src/metadata/mod.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index 2111595d5c7..99dccee5a3a 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -718,21 +718,11 @@ impl RafsSuper { // Old converters extracts bootstraps from data blobs with inlined bootstrap // use blob digest as the bootstrap file name. The last blob in the blob table from // the bootstrap has wrong blod id, so we need to fix it. - let mut fixed = false; let blobs = rs.superblock.get_blob_infos(); for blob in blobs.iter() { // Fix blob id for new images with old converters. if blob.has_feature(BlobFeatures::INLINED_FS_META) { blob.set_blob_id_from_meta_path(path.as_ref())?; - fixed = true; - } - } - if !fixed && !blob_accessible && !blobs.is_empty() { - // Fix blob id for old images with old converters. - let last = blobs.len() - 1; - let blob = &blobs[last]; - if !blob.has_feature(BlobFeatures::CAP_TAR_TOC) { - rs.set_blob_id_from_meta_path(path.as_ref())?; } } }