Merge pull request #1451 from anyproto/go-2746-export-structure

GO-2746: add export structure

core/block/export/export.go

@@ -55,8 +55,14 @@ import (
 const CName = "export"
 
 const (
-	tempFileName   = "temp_anytype_backup"
-	spaceDirectory = "spaces"
+	tempFileName              = "temp_anytype_backup"
+	spaceDirectory            = "spaces"
+	typesDirectory            = "types"
+	objectsDirectory          = "objects"
+	relationsDirectory        = "relations"
+	relationsOptionsDirectory = "relationsOptions"
+	templatesDirectory        = "templates"
+	filesObjects              = "filesObjects"
 )
 
 var log = logging.Logger("anytype-mw-export")
@@ -461,8 +467,8 @@ func (e *export) writeMultiDoc(ctx context.Context,
 	return
 }
 
-func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docID string) (err error) {
-	return cache.Do(e.picker, docID, func(b sb.SmartBlock) error {
+func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docId string) (err error) {
+	return cache.Do(e.picker, docId, func(b sb.SmartBlock) error {
 		st := b.NewState()
 		if pbtypes.GetBool(st.CombinedDetails(), bundle.RelationKeyIsDeleted.String()) {
 			return nil
@@ -491,12 +497,13 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques
 		}
 		conv.SetKnownDocs(docInfo)
 		result := conv.Convert(b.Type().ToProto())
-		filename := e.provideFileName(docID, req.SpaceId, conv, st)
+		var filename string
 		if req.Format == model.Export_Markdown {
-			filename = e.provideMarkdownName(st, wr, docID, conv, req.SpaceId)
-		}
-		if docID == b.Space().DerivedIDs().Home {
+			filename = e.makeMarkdownName(st, wr, docId, conv, req.SpaceId)
+		} else if docId == b.Space().DerivedIDs().Home {
 			filename = "index" + conv.Ext()
+		} else {
+			filename = e.makeFileName(docId, req.SpaceId, conv, st, b.Type())
 		}
 		lastModifiedDate := pbtypes.GetInt64(st.LocalDetails(), bundle.RelationKeyLastModifiedDate.String())
 		if err = wr.WriteFile(filename, bytes.NewReader(result), lastModifiedDate); err != nil {
@@ -506,28 +513,48 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques
 	})
 }
 
-func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, conv converter.Converter, spaceId string) string {
+func (e *export) makeMarkdownName(s *state.State, wr writer, docID string, conv converter.Converter, spaceId string) string {
 	name := pbtypes.GetString(s.Details(), bundle.RelationKeyName.String())
 	if name == "" {
 		name = s.Snippet()
 	}
 	path := ""
+	// space can be empty in case user want to export all spaces
 	if spaceId == "" {
 		spaceId := pbtypes.GetString(s.LocalDetails(), bundle.RelationKeySpaceId.String())
 		path = filepath.Join(spaceDirectory, spaceId)
 	}
 	return wr.Namer().Get(path, docID, name, conv.Ext())
 }
 
-func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State) string {
-	filename := docID + conv.Ext()
+func (e *export) makeFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string {
+	dir := e.provideFileDirectory(blockType)
+	filename := filepath.Join(dir, docId+conv.Ext())
+	// space can be empty in case user want to export all spaces
 	if spaceId == "" {
 		spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String())
 		filename = filepath.Join(spaceDirectory, spaceId, filename)
 	}
 	return filename
 }
 
+func (e *export) provideFileDirectory(blockType smartblock.SmartBlockType) string {
+	switch blockType {
+	case smartblock.SmartBlockTypeRelation:
+		return relationsDirectory
+	case smartblock.SmartBlockTypeRelationOption:
+		return relationsOptionsDirectory
+	case smartblock.SmartBlockTypeObjectType:
+		return typesDirectory
+	case smartblock.SmartBlockTypeTemplate:
+		return templatesDirectory
+	case smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeFileObject:
+		return filesObjects
+	default:
+		return objectsDirectory
+	}
+}
+
 func (e *export) saveFile(ctx context.Context, wr writer, fileObject sb.SmartBlock, exportAllSpaces bool) (fileName string, err error) {
 	fullId := domain.FullFileId{
 		SpaceId: fileObject.Space().Id(),

core/block/export/export_test.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/anyproto/anytype-heart/core/block/cache/mock_cache"
 	"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
+	"github.com/anyproto/anytype-heart/core/block/editor/state"
+	"github.com/anyproto/anytype-heart/core/converter/pbjson"
 	"github.com/anyproto/anytype-heart/core/domain"
 	"github.com/anyproto/anytype-heart/pb"
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
@@ -495,3 +497,68 @@ func Test_docsForExport(t *testing.T) {
 		assert.Equal(t, 2, len(docsForExport))
 	})
 }
+
+func Test_provideFileName(t *testing.T) {
+	t.Run("file dir for relation", func(t *testing.T) {
+		// given
+		e := &export{}
+
+		// when
+		fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation)
+
+		// then
+		assert.Equal(t, relationsDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
+	})
+	t.Run("file dir for relation option", func(t *testing.T) {
+		// given
+		e := &export{}
+
+		// when
+		fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption)
+
+		// then
+		assert.Equal(t, relationsOptionsDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
+	})
+	t.Run("file dir for types", func(t *testing.T) {
+		// given
+		e := &export{}
+
+		// when
+		fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType)
+
+		// then
+		assert.Equal(t, typesDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
+	})
+	t.Run("file dir for objects", func(t *testing.T) {
+		// given
+		e := &export{}
+
+		// when
+		fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage)
+
+		// then
+		assert.Equal(t, objectsDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
+	})
+	t.Run("file dir for files objects", func(t *testing.T) {
+		// given
+		e := &export{}
+
+		// when
+		fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject)
+
+		// then
+		assert.Equal(t, filesObjects+string(filepath.Separator)+"docId.pb.json", fileName)
+	})
+	t.Run("space is not provided", func(t *testing.T) {
+		// given
+		e := &export{}
+		st := state.NewDoc("root", nil).(*state.State)
+		st.SetDetail(bundle.RelationKeySpaceId.String(), pbtypes.String("spaceId"))
+
+		// when
+		fileName := e.makeFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject)
+
+		// then
+		assert.Equal(t, spaceDirectory+string(filepath.Separator)+"spaceId"+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName)
+	})
+}

core/block/export/writer.go

@@ -58,6 +58,11 @@ func (d *dirWriter) Path() string {
 }
 
 func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) {
+	dir := filepath.Dir(filename)
+	err = os.MkdirAll(filepath.Join(d.path, dir), 0700)
+	if err != nil {
+		return err
+	}
 	filename = path.Join(d.path, filename)
 	f, err := os.Create(filename)
 	if err != nil {

tests/integration/export_test.go

@@ -84,8 +84,18 @@ func TestExportFiles(t *testing.T) {
 
 		var foundPbFiles int
 		for _, entry := range entries {
-			if filepath.Ext(entry.Name()) == ".pb" {
-				foundPbFiles++
+			if entry.IsDir() {
+				files, err := os.ReadDir(filepath.Join(exportPath, entry.Name()))
+				require.NoError(t, err)
+				for _, file := range files {
+					if filepath.Ext(file.Name()) == ".pb" {
+						foundPbFiles++
+					}
+				}
+			} else {
+				if filepath.Ext(entry.Name()) == ".pb" {
+					foundPbFiles++
+				}
 			}
 		}
 		// 4 objects total: Page object + Page type + File object