Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GO-2746: add export structure #1451

Merged
merged 6 commits into from
Aug 13, 2024
Merged

GO-2746: add export structure #1451

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9cf7e4e
GO-2746: add export structure
AnastasiaShemyakinskaya Aug 6, 2024
669c280
GO-2746: fix comments
AnastasiaShemyakinskaya Aug 7, 2024
5e1e9c5
GO-2746 Merge branch 'main' of github.com:anyproto/anytype-heart into…
AnastasiaShemyakinskaya Aug 7, 2024
533f2ec
GO-2746: refactoring
AnastasiaShemyakinskaya Aug 8, 2024
4501c58
GO-2746 Merge branch 'main' of github.com:anyproto/anytype-heart into…
AnastasiaShemyakinskaya Aug 8, 2024
b4f8757
GO-2746: add comments
AnastasiaShemyakinskaya Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 33 additions & 9 deletions core/block/export/export.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,14 @@ import (
const CName = "export"

const (
tempFileName = "temp_anytype_backup"
spaceDirectory = "spaces"
tempFileName = "temp_anytype_backup"
spaceDirectory = "spaces"
typesDirectory = "types"
objectsDirectory = "objects"
relationsDirectory = "relations"
relationsOptionsDirectory = "relationsOptions"
templatesDirectory = "templates"
filesObjects = "filesObjects"
)

var log = logging.Logger("anytype-mw-export")
Expand Down Expand Up @@ -461,8 +467,8 @@ func (e *export) writeMultiDoc(ctx context.Context,
return
}

func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docID string) (err error) {
return cache.Do(e.picker, docID, func(b sb.SmartBlock) error {
func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docId string) (err error) {
return cache.Do(e.picker, docId, func(b sb.SmartBlock) error {
st := b.NewState()
if pbtypes.GetBool(st.CombinedDetails(), bundle.RelationKeyIsDeleted.String()) {
return nil
Expand Down Expand Up @@ -491,11 +497,11 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques
}
conv.SetKnownDocs(docInfo)
result := conv.Convert(b.Type().ToProto())
filename := e.provideFileName(docID, req.SpaceId, conv, st)
filename := e.provideFileName(docId, req.SpaceId, conv, st, b.Type())
if req.Format == model.Export_Markdown {
filename = e.provideMarkdownName(st, wr, docID, conv, req.SpaceId)
filename = e.provideMarkdownName(st, wr, docId, conv, req.SpaceId)
}
if docID == b.Space().DerivedIDs().Home {
if docId == b.Space().DerivedIDs().Home {
filename = "index" + conv.Ext()
}
lastModifiedDate := pbtypes.GetInt64(st.LocalDetails(), bundle.RelationKeyLastModifiedDate.String())
Expand All @@ -519,15 +525,33 @@ func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, co
return wr.Namer().Get(path, docID, name, conv.Ext())
}

func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State) string {
filename := docID + conv.Ext()
func (e *export) provideFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string {
dir := e.provideFileDirectory(blockType)
filename := filepath.Join(dir, docId+conv.Ext())
if spaceId == "" {
spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can it be the case that spaceId is nil? What happens here, why if spaceId == "" we do something completely different? Maybe It shouldn't be inside this function? Can you explain please?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It can be empty, if user want to export all spaces. We build file directory based on this value, so I guess it makes sense to check it here 🧐

filename = filepath.Join(spaceDirectory, spaceId, filename)
}
return filename
}

func (e *export) provideFileDirectory(blockType smartblock.SmartBlockType) string {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

call it just fileDirectory maybe? and then fileName?

switch blockType {
case smartblock.SmartBlockTypeRelation:
return relationsDirectory
case smartblock.SmartBlockTypeRelationOption:
return relationsOptionsDirectory
case smartblock.SmartBlockTypeObjectType:
return typesDirectory
case smartblock.SmartBlockTypeTemplate:
return templatesDirectory
case smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeFileObject:
return filesObjects
default:
return objectsDirectory
}
}

func (e *export) saveFile(ctx context.Context, wr writer, fileObject sb.SmartBlock, exportAllSpaces bool) (fileName string, err error) {
fullId := domain.FullFileId{
SpaceId: fileObject.Space().Id(),
Expand Down
67 changes: 67 additions & 0 deletions core/block/export/export_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (

"github.com/anyproto/anytype-heart/core/block/cache/mock_cache"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
"github.com/anyproto/anytype-heart/core/block/editor/state"
"github.com/anyproto/anytype-heart/core/converter/pbjson"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
Expand Down Expand Up @@ -495,3 +497,68 @@ func Test_docsForExport(t *testing.T) {
assert.Equal(t, 2, len(docsForExport))
})
}

func Test_provideFileName(t *testing.T) {
t.Run("file dir for relation", func(t *testing.T) {
// given
e := &export{}

// when
fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation)

// then
assert.Equal(t, relationsDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
})
t.Run("file dir for relation option", func(t *testing.T) {
// given
e := &export{}

// when
fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption)

// then
assert.Equal(t, relationsOptionsDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
})
t.Run("file dir for types", func(t *testing.T) {
// given
e := &export{}

// when
fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType)

// then
assert.Equal(t, typesDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
})
t.Run("file dir for objects", func(t *testing.T) {
// given
e := &export{}

// when
fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage)

// then
assert.Equal(t, objectsDirectory+string(filepath.Separator)+"docId.pb.json", fileName)
})
t.Run("file dir for files objects", func(t *testing.T) {
// given
e := &export{}

// when
fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject)

// then
assert.Equal(t, filesObjects+string(filepath.Separator)+"docId.pb.json", fileName)
})
t.Run("space is not provided", func(t *testing.T) {
// given
e := &export{}
st := state.NewDoc("root", nil).(*state.State)
st.SetDetail(bundle.RelationKeySpaceId.String(), pbtypes.String("spaceId"))

// when
fileName := e.provideFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject)

// then
assert.Equal(t, spaceDirectory+string(filepath.Separator)+"spaceId"+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName)
})
}
5 changes: 5 additions & 0 deletions core/block/export/writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ func (d *dirWriter) Path() string {
}

func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) {
dir := filepath.Dir(filename)
err = os.MkdirAll(filepath.Join(d.path, dir), 0700)
if err != nil {
return err
}
filename = path.Join(d.path, filename)
f, err := os.Create(filename)
if err != nil {
Expand Down
14 changes: 12 additions & 2 deletions tests/integration/export_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,18 @@ func TestExportFiles(t *testing.T) {

var foundPbFiles int
for _, entry := range entries {
if filepath.Ext(entry.Name()) == ".pb" {
foundPbFiles++
if entry.IsDir() {
files, err := os.ReadDir(filepath.Join(exportPath, entry.Name()))
require.NoError(t, err)
for _, file := range files {
if filepath.Ext(file.Name()) == ".pb" {
foundPbFiles++
}
}
} else {
if filepath.Ext(entry.Name()) == ".pb" {
foundPbFiles++
}
}
}
// 4 objects total: Page object + Page type + File object
Expand Down
Loading