diff --git a/core/block/export/export.go b/core/block/export/export.go index 549a3d9bc9..d0a1a95c63 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -55,8 +55,14 @@ import ( const CName = "export" const ( - tempFileName = "temp_anytype_backup" - spaceDirectory = "spaces" + tempFileName = "temp_anytype_backup" + spaceDirectory = "spaces" + typesDirectory = "types" + objectsDirectory = "objects" + relationsDirectory = "relations" + relationsOptionsDirectory = "relationsOptions" + templatesDirectory = "templates" + filesObjects = "filesObjects" ) var log = logging.Logger("anytype-mw-export") @@ -461,8 +467,8 @@ func (e *export) writeMultiDoc(ctx context.Context, return } -func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docID string) (err error) { - return cache.Do(e.picker, docID, func(b sb.SmartBlock) error { +func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docId string) (err error) { + return cache.Do(e.picker, docId, func(b sb.SmartBlock) error { st := b.NewState() if pbtypes.GetBool(st.CombinedDetails(), bundle.RelationKeyIsDeleted.String()) { return nil @@ -491,12 +497,13 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques } conv.SetKnownDocs(docInfo) result := conv.Convert(b.Type().ToProto()) - filename := e.provideFileName(docID, req.SpaceId, conv, st) + var filename string if req.Format == model.Export_Markdown { - filename = e.provideMarkdownName(st, wr, docID, conv, req.SpaceId) - } - if docID == b.Space().DerivedIDs().Home { + filename = e.makeMarkdownName(st, wr, docId, conv, req.SpaceId) + } else if docId == b.Space().DerivedIDs().Home { filename = "index" + conv.Ext() + } else { + filename = e.makeFileName(docId, req.SpaceId, conv, st, b.Type()) } lastModifiedDate := pbtypes.GetInt64(st.LocalDetails(), bundle.RelationKeyLastModifiedDate.String()) if err = wr.WriteFile(filename, bytes.NewReader(result), lastModifiedDate); err != nil { @@ -506,12 +513,13 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques }) } -func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, conv converter.Converter, spaceId string) string { +func (e *export) makeMarkdownName(s *state.State, wr writer, docID string, conv converter.Converter, spaceId string) string { name := pbtypes.GetString(s.Details(), bundle.RelationKeyName.String()) if name == "" { name = s.Snippet() } path := "" + // space can be empty in case user want to export all spaces if spaceId == "" { spaceId := pbtypes.GetString(s.LocalDetails(), bundle.RelationKeySpaceId.String()) path = filepath.Join(spaceDirectory, spaceId) @@ -519,8 +527,10 @@ func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, co return wr.Namer().Get(path, docID, name, conv.Ext()) } -func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State) string { - filename := docID + conv.Ext() +func (e *export) makeFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { + dir := e.provideFileDirectory(blockType) + filename := filepath.Join(dir, docId+conv.Ext()) + // space can be empty in case user want to export all spaces if spaceId == "" { spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String()) filename = filepath.Join(spaceDirectory, spaceId, filename) @@ -528,6 +538,23 @@ func (e *export) provideFileName(docID, spaceId string, conv converter.Converter return filename } +func (e *export) provideFileDirectory(blockType smartblock.SmartBlockType) string { + switch blockType { + case smartblock.SmartBlockTypeRelation: + return relationsDirectory + case smartblock.SmartBlockTypeRelationOption: + return relationsOptionsDirectory + case smartblock.SmartBlockTypeObjectType: + return typesDirectory + case smartblock.SmartBlockTypeTemplate: + return templatesDirectory + case smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeFileObject: + return filesObjects + default: + return objectsDirectory + } +} + func (e *export) saveFile(ctx context.Context, wr writer, fileObject sb.SmartBlock, exportAllSpaces bool) (fileName string, err error) { fullId := domain.FullFileId{ SpaceId: fileObject.Space().Id(), diff --git a/core/block/export/export_test.go b/core/block/export/export_test.go index c9b4270329..c5b697e940 100644 --- a/core/block/export/export_test.go +++ b/core/block/export/export_test.go @@ -11,6 +11,8 @@ import ( "github.com/anyproto/anytype-heart/core/block/cache/mock_cache" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/converter/pbjson" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -495,3 +497,68 @@ func Test_docsForExport(t *testing.T) { assert.Equal(t, 2, len(docsForExport)) }) } + +func Test_provideFileName(t *testing.T) { + t.Run("file dir for relation", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation) + + // then + assert.Equal(t, relationsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for relation option", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption) + + // then + assert.Equal(t, relationsOptionsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for types", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType) + + // then + assert.Equal(t, typesDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for objects", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage) + + // then + assert.Equal(t, objectsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for files objects", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject) + + // then + assert.Equal(t, filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("space is not provided", func(t *testing.T) { + // given + e := &export{} + st := state.NewDoc("root", nil).(*state.State) + st.SetDetail(bundle.RelationKeySpaceId.String(), pbtypes.String("spaceId")) + + // when + fileName := e.makeFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject) + + // then + assert.Equal(t, spaceDirectory+string(filepath.Separator)+"spaceId"+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) + }) +} diff --git a/core/block/export/writer.go b/core/block/export/writer.go index df5b87bf1d..cc89bbb25f 100644 --- a/core/block/export/writer.go +++ b/core/block/export/writer.go @@ -58,6 +58,11 @@ func (d *dirWriter) Path() string { } func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) { + dir := filepath.Dir(filename) + err = os.MkdirAll(filepath.Join(d.path, dir), 0700) + if err != nil { + return err + } filename = path.Join(d.path, filename) f, err := os.Create(filename) if err != nil { diff --git a/tests/integration/export_test.go b/tests/integration/export_test.go index 8971a5ccea..f2ede4fd8f 100644 --- a/tests/integration/export_test.go +++ b/tests/integration/export_test.go @@ -84,8 +84,18 @@ func TestExportFiles(t *testing.T) { var foundPbFiles int for _, entry := range entries { - if filepath.Ext(entry.Name()) == ".pb" { - foundPbFiles++ + if entry.IsDir() { + files, err := os.ReadDir(filepath.Join(exportPath, entry.Name())) + require.NoError(t, err) + for _, file := range files { + if filepath.Ext(file.Name()) == ".pb" { + foundPbFiles++ + } + } + } else { + if filepath.Ext(entry.Name()) == ".pb" { + foundPbFiles++ + } } } // 4 objects total: Page object + Page type + File object