From 9cf7e4e18c5cdcb9bbbcc7de8157621cf9a1aab9 Mon Sep 17 00:00:00 2001 From: AnastasiaShemyakinskaya Date: Tue, 6 Aug 2024 14:47:36 +0200 Subject: [PATCH] GO-2746: add export structure Signed-off-by: AnastasiaShemyakinskaya --- core/block/export/export.go | 34 +++++++++++++--- core/block/export/export_test.go | 67 ++++++++++++++++++++++++++++++++ core/block/export/writer.go | 7 ++++ tests/integration/export_test.go | 14 ++++++- 4 files changed, 115 insertions(+), 7 deletions(-) diff --git a/core/block/export/export.go b/core/block/export/export.go index 8f4ddb83e1..a7c47b5a4a 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -55,8 +55,14 @@ import ( const CName = "export" const ( - tempFileName = "temp_anytype_backup" - spaceDirectory = "spaces" + tempFileName = "temp_anytype_backup" + spaceDirectory = "spaces" + typesDirectory = "types" + objectsDirectory = "objects" + relationsDirectory = "relations" + relationsOptionsDirectory = "relationsOptions" + templatesDirectory = "templates" + filesObjects = "filesObjects" ) var log = logging.Logger("anytype-mw-export") @@ -491,7 +497,7 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques } conv.SetKnownDocs(docInfo) result := conv.Convert(b.Type().ToProto()) - filename := e.provideFileName(docID, req.SpaceId, conv, st) + filename := e.provideFileName(docID, req.SpaceId, conv, st, b.Type()) if req.Format == model.Export_Markdown { filename = e.provideMarkdownName(st, wr, docID, conv, req.SpaceId) } @@ -519,8 +525,9 @@ func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, co return wr.Namer().Get(path, docID, name, conv.Ext()) } -func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State) string { - filename := docID + conv.Ext() +func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { + dir := e.provideFileDirectory(blockType) + filename := filepath.Join(dir, docID+conv.Ext()) if spaceId == "" { spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String()) filename = filepath.Join(spaceDirectory, spaceId, filename) @@ -528,6 +535,23 @@ func (e *export) provideFileName(docID, spaceId string, conv converter.Converter return filename } +func (e *export) provideFileDirectory(blockType smartblock.SmartBlockType) string { + switch blockType { + case smartblock.SmartBlockTypeRelation: + return relationsDirectory + case smartblock.SmartBlockTypeRelationOption: + return relationsOptionsDirectory + case smartblock.SmartBlockTypeObjectType: + return typesDirectory + case smartblock.SmartBlockTypeTemplate: + return templatesDirectory + case smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeFileObject: + return filesObjects + default: + return objectsDirectory + } +} + func (e *export) saveFile(ctx context.Context, wr writer, fileObject sb.SmartBlock, exportAllSpaces bool) (fileName string, err error) { fullId := domain.FullFileId{ SpaceId: fileObject.Space().Id(), diff --git a/core/block/export/export_test.go b/core/block/export/export_test.go index c9b4270329..bd616af079 100644 --- a/core/block/export/export_test.go +++ b/core/block/export/export_test.go @@ -11,6 +11,8 @@ import ( "github.com/anyproto/anytype-heart/core/block/cache/mock_cache" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/converter/pbjson" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -495,3 +497,68 @@ func Test_docsForExport(t *testing.T) { assert.Equal(t, 2, len(docsForExport)) }) } + +func Test_provideFileName(t *testing.T) { + t.Run("file dir for relation", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation) + + // then + assert.Equal(t, relationsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for relation option", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption) + + // then + assert.Equal(t, relationsOptionsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for types", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType) + + // then + assert.Equal(t, typesDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for objects", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage) + + // then + assert.Equal(t, objectsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for files objects", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject) + + // then + assert.Equal(t, filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("space is not provided", func(t *testing.T) { + // given + e := &export{} + st := state.NewDoc("root", nil).(*state.State) + st.SetDetail(bundle.RelationKeySpaceId.String(), pbtypes.String("spaceId")) + + // when + fileName := e.provideFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject) + + // then + assert.Equal(t, spaceDirectory+string(filepath.Separator)+"spaceId"+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) + }) +} diff --git a/core/block/export/writer.go b/core/block/export/writer.go index bbfdd95a76..8493cd4192 100644 --- a/core/block/export/writer.go +++ b/core/block/export/writer.go @@ -58,6 +58,13 @@ func (d *dirWriter) Path() string { } func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) { + dir := filepath.Dir(filename) + if dir != "" { + err = os.MkdirAll(filepath.Join(d.path, dir), 0700) + if err != nil { + return err + } + } filename = path.Join(d.path, filename) f, err := os.Create(filename) if err != nil { diff --git a/tests/integration/export_test.go b/tests/integration/export_test.go index 8971a5ccea..f2ede4fd8f 100644 --- a/tests/integration/export_test.go +++ b/tests/integration/export_test.go @@ -84,8 +84,18 @@ func TestExportFiles(t *testing.T) { var foundPbFiles int for _, entry := range entries { - if filepath.Ext(entry.Name()) == ".pb" { - foundPbFiles++ + if entry.IsDir() { + files, err := os.ReadDir(filepath.Join(exportPath, entry.Name())) + require.NoError(t, err) + for _, file := range files { + if filepath.Ext(file.Name()) == ".pb" { + foundPbFiles++ + } + } + } else { + if filepath.Ext(entry.Name()) == ".pb" { + foundPbFiles++ + } } } // 4 objects total: Page object + Page type + File object