GO-4162 corrupted links in markdown import

core/block/import/html/converter.go

@@ -189,7 +189,7 @@ func (h *HTML) updateFilesInLinks(block *model.Block, filesSource source.Source,
 			if newFileName, createFileBlock, err = common.ProvideFileName(mark.Param, filesSource, path, h.tempDirProvider); err == nil {
 				mark.Param = newFileName
 				if createFileBlock {
-					anymark.ConvertTextToFile(block)
+					block.Content = anymark.ConvertTextToFile(mark.Param)
 					break
 				}
 				continue

core/block/import/markdown/anymark/anyblocks.go

@@ -92,10 +92,10 @@ func provideCodeBlock(textArr []string, language string, id string) *model.Block
 	}
 }
 
-func ConvertTextToFile(block *model.Block) {
+func ConvertTextToFile(filePath string) *model.BlockContentOfFile {
 	// "svg" excluded
-	if block.GetText().GetMarks().Marks[0].Param == "" {
-		return
+	if filePath == "" {
+		return nil
 	}
 
 	imageFormats := []string{"jpg", "jpeg", "png", "gif", "webp"}
@@ -104,7 +104,7 @@ func ConvertTextToFile(block *model.Block) {
 	pdfFormat := "pdf"
 
 	fileType := model.BlockContentFile_File
-	fileExt := filepath.Ext(block.GetText().GetMarks().Marks[0].Param)
+	fileExt := filepath.Ext(filePath)
 	if fileExt != "" {
 		fileExt = fileExt[1:]
 		for _, ext := range imageFormats {
@@ -131,14 +131,13 @@ func ConvertTextToFile(block *model.Block) {
 		if strings.EqualFold(fileExt, pdfFormat) {
 			fileType = model.BlockContentFile_PDF
 		}
-
-		block.Content = &model.BlockContentOfFile{
-			File: &model.BlockContentFile{
-				Name:  block.GetText().GetMarks().Marks[0].Param,
-				State: model.BlockContentFile_Empty,
-				Type:  fileType,
-			},
-		}
+	}
+	return &model.BlockContentOfFile{
+		File: &model.BlockContentFile{
+			Name:  filePath,
+			State: model.BlockContentFile_Empty,
+			Type:  fileType,
+		},
 	}
 }
 

core/block/import/markdown/blockconverter.go

@@ -98,35 +98,72 @@ func (m *mdConverter) processBlocks(shortPath string, file *FileInfo, files map[
 
 func (m *mdConverter) processTextBlock(block *model.Block, files map[string]*FileInfo) {
 	txt := block.GetText()
-	if txt != nil && txt.Marks != nil && len(txt.Marks.Marks) == 1 &&
-		txt.Marks.Marks[0].Type == model.BlockContentTextMark_Link {
-		link := txt.Marks.Marks[0].Param
-		wholeLineLink := m.isWholeLineLink(txt)
-		ext := filepath.Ext(link)
-
-		// todo: bug with multiple markup links in arow when the first is external
-		if file := files[link]; file != nil {
-			if strings.EqualFold(ext, ".csv") {
-				m.processCSVFileLink(block, files, link, wholeLineLink)
+	if txt != nil && txt.Marks != nil {
+		if len(txt.Marks.Marks) == 1 && txt.Marks.Marks[0].Type == model.BlockContentTextMark_Link {
+			m.handleSingleMark(block, files)
+		} else {
+			m.handleMultipleMarks(block, files)
+		}
+	}
+}
+
+func (m *mdConverter) handleSingleMark(block *model.Block, files map[string]*FileInfo) {
+	txt := block.GetText()
+	link := txt.Marks.Marks[0].Param
+	wholeLineLink := m.isWholeLineLink(txt.Text, txt.Marks.Marks[0])
+	ext := filepath.Ext(link)
+	if file := files[link]; file != nil {
+		if strings.EqualFold(ext, ".csv") {
+			m.processCSVFileLink(block, files, link, wholeLineLink)
+			return
+		}
+		if strings.EqualFold(ext, ".md") {
+			// only convert if this is the only link in the row
+			m.convertToAnytypeLinkBlock(block, wholeLineLink)
+		} else {
+			block.Content = anymark.ConvertTextToFile(txt.Marks.Marks[0].Param)
+		}
+		file.HasInboundLinks = true
+	} else if wholeLineLink {
+		block.Content = m.convertTextToBookmark(txt.Marks.Marks[0].Param)
+	}
+}
+
+func (m *mdConverter) handleMultipleMarks(block *model.Block, files map[string]*FileInfo) {
+	txt := block.GetText()
+	for _, mark := range txt.Marks.Marks {
+		if mark.Type == model.BlockContentTextMark_Link {
+			if stop := m.handleSingleLinkMark(block, files, mark, txt); stop {
 				return
 			}
-			if strings.EqualFold(ext, ".md") {
-				// only convert if this is the only link in the row
-				m.convertToAnytypeLinkBlock(block, wholeLineLink)
-			} else {
-				anymark.ConvertTextToFile(block)
-			}
-			file.HasInboundLinks = true
-		} else if wholeLineLink {
-			m.convertTextToBookmark(block)
 		}
 	}
 }
 
-func (m *mdConverter) isWholeLineLink(txt *model.BlockContentText) bool {
+func (m *mdConverter) handleSingleLinkMark(block *model.Block, files map[string]*FileInfo, mark *model.BlockContentTextMark, txt *model.BlockContentText) bool {
+	link := mark.Param
+	ext := filepath.Ext(link)
+	if file := files[link]; file != nil {
+		file.HasInboundLinks = true
+		if strings.EqualFold(ext, ".md") || strings.EqualFold(ext, ".csv") {
+			mark.Type = model.BlockContentTextMark_Mention
+			return false
+		}
+		if m.isWholeLineLink(txt.Text, mark) {
+			block.Content = anymark.ConvertTextToFile(mark.Param)
+			return true
+		}
+	} else if m.isWholeLineLink(txt.Text, mark) {
+		block.Content = m.convertTextToBookmark(mark.Param)
+		return true
+	}
+	return false
+}
+
+func (m *mdConverter) isWholeLineLink(text string, marks *model.BlockContentTextMark) bool {
 	var wholeLineLink bool
-	textRunes := []rune(txt.Text)
-	var from, to = int(txt.Marks.Marks[0].Range.From), int(txt.Marks.Marks[0].Range.To)
+	textRunes := []rune(text)
+	var from, to = int(marks.Range.From), int(marks.Range.To)
 	if from == 0 || (from < len(textRunes) && len(strings.TrimSpace(string(textRunes[0:from]))) == 0) {
 		if to >= len(textRunes) || len(strings.TrimSpace(string(textRunes[to:]))) == 0 {
 			wholeLineLink = true
@@ -201,14 +238,14 @@ func (m *mdConverter) convertTextToPageLink(block *model.Block) {
 	}
 }
 
-func (m *mdConverter) convertTextToBookmark(block *model.Block) {
-	if err := uri.ValidateURI(block.GetText().Marks.Marks[0].Param); err != nil {
-		return
+func (m *mdConverter) convertTextToBookmark(url string) *model.BlockContentOfBookmark {
+	if err := uri.ValidateURI(url); err != nil {
+		return nil
 	}
 
-	block.Content = &model.BlockContentOfBookmark{
+	return &model.BlockContentOfBookmark{
 		Bookmark: &model.BlockContentBookmark{
-			Url: block.GetText().Marks.Marks[0].Param,
+			Url: url,
 		},
 	}
 }

core/block/import/markdown/blockconverter_test.go

@@ -39,7 +39,7 @@ func Test_processFiles(t *testing.T) {
 		files := converter.processFiles(absolutePath, common.NewError(pb.RpcObjectImportRequest_IGNORE_ERRORS), source)
 
 		// then
-		assert.Len(t, files, 3)
+		assert.Len(t, files, 6)
 
 		pdfFilePath := filepath.Join(absolutePath, "test.pdf")
 		assert.Contains(t, files, pdfFilePath)
@@ -71,7 +71,7 @@ func Test_processFiles(t *testing.T) {
 		files := converter.processFiles(absolutePath, common.NewError(pb.RpcObjectImportRequest_IGNORE_ERRORS), source)
 
 		// then
-		assert.Len(t, files, 1)
+		assert.Len(t, files, 4)
 
 		pdfFilePath := filepath.Join(absolutePath, "test.pdf")
 		assert.NotContains(t, files, pdfFilePath)

core/block/import/markdown/import.go

@@ -140,8 +140,8 @@ func (m *Markdown) getSnapshotsAndRootObjectsIds(
 		m.processImportStep(pathsCount, files, progress, allErrors, details, m.setNewID) ||
 		m.processImportStep(pathsCount, files, progress, allErrors, details, m.addLinkToObjectBlocks) ||
 		m.processImportStep(pathsCount, files, progress, allErrors, details, m.linkPagesWithRootFile) ||
-		m.processImportStep(pathsCount, files, progress, allErrors, details, m.fillEmptyBlocks) ||
 		m.processImportStep(pathsCount, files, progress, allErrors, details, m.addLinkBlocks) ||
+		m.processImportStep(pathsCount, files, progress, allErrors, details, m.fillEmptyBlocks) ||
 		m.processImportStep(pathsCount, files, progress, allErrors, details, m.addChildBlocks) {
 		return nil, nil
 	}
@@ -448,7 +448,7 @@ func (m *Markdown) addChildBlocks(files map[string]*FileInfo, progress process.P
 			continue
 		}
 
-		var childrenIds = make([]string, len(file.ParsedBlocks))
+		childrenIds := make([]string, 0, len(file.ParsedBlocks))
 		for _, b := range file.ParsedBlocks {
 			if isChildBlock(childBlocks, b) {
 				continue

core/block/import/markdown/import_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/anyproto/anytype-heart/core/block/process"
 	"github.com/anyproto/anytype-heart/pb"
 	"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
+	"github.com/anyproto/anytype-heart/tests/blockbuilder"
 )
 
 func TestMarkdown_GetSnapshots(t *testing.T) {
@@ -75,6 +76,146 @@ func TestMarkdown_GetSnapshots(t *testing.T) {
 		assert.Nil(t, sn)
 		assert.True(t, err.IsNoObjectToImportError(1))
 	})
+	t.Run("import file with links", func(t *testing.T) {
+		// given
+		tempDirProvider := &MockTempDir{}
+		converter := newMDConverter(tempDirProvider)
+		h := &Markdown{blockConverter: converter}
+		p := process.NewProgress(pb.ModelProcess_Import)
+
+		// when
+		sn, err := h.GetSnapshots(context.Background(), &pb.RpcObjectImportRequest{
+			Params: &pb.RpcObjectImportRequestParamsOfMarkdownParams{
+				MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: []string{"testdata"}},
+			},
+			Type: model.Import_Markdown,
+			Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS,
+		}, p)
+
+		// then
+		assert.Nil(t, err)
+		assert.NotNil(t, sn)
+		assert.Len(t, sn.Snapshots, 4)
+
+		fileNameToObjectId := make(map[string]string, len(sn.Snapshots))
+		for _, snapshot := range sn.Snapshots {
+			fileNameToObjectId[snapshot.FileName] = snapshot.Id
+		}
+		var found bool
+		expectedPath := filepath.Join("testdata", "links.md")
+		rootId := fileNameToObjectId[expectedPath]
+		want := buildExpectedTree(fileNameToObjectId, tempDirProvider, rootId)
+		for _, snapshot := range sn.Snapshots {
+			if snapshot.FileName == expectedPath {
+				found = true
+				blockbuilder.AssertTreesEqual(t, want.Build(), snapshot.Snapshot.Data.Blocks)
+			}
+		}
+		assert.True(t, found)
+	})
+}
+
+func buildExpectedTree(fileNameToObjectId map[string]string, provider *MockTempDir, rootId string) *blockbuilder.Block {
+	fileMdPath := filepath.Join("testdata", "file.md")
+	testMdPath := filepath.Join("testdata", "test.md")
+	testCsvPath := filepath.Join("testdata", "test.csv")
+	testTxtPath := filepath.Join("testdata", "test.txt")
+	want := blockbuilder.Root(
+		blockbuilder.ID(rootId),
+		blockbuilder.Children(
+			blockbuilder.Text("File does not exist test1", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 20, To: 25},
+					Type:  model.BlockContentTextMark_Link,
+					Param: fileMdPath,
+				},
+			}})),
+			blockbuilder.Text("Test link to page test2", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 18, To: 23},
+					Type:  model.BlockContentTextMark_Mention,
+					Param: fileNameToObjectId[testMdPath],
+				},
+			}})),
+			blockbuilder.File("", blockbuilder.FileName(provider.TempDir()+testTxtPath), blockbuilder.FileType(model.BlockContentFile_File)),
+			blockbuilder.Text("Test link to csv test4", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 17, To: 22},
+					Type:  model.BlockContentTextMark_Mention,
+					Param: fileNameToObjectId[testCsvPath],
+				},
+			}})),
+			blockbuilder.Text("File does not exist with bold mark test1", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 35, To: 40},
+					Type:  model.BlockContentTextMark_Link,
+					Param: fileMdPath,
+				},
+				{
+					Range: &model.Range{From: 35, To: 40},
+					Type:  model.BlockContentTextMark_Bold,
+				},
+			}})),
+			blockbuilder.Text("Test link to page with bold mark test2", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 33, To: 38},
+					Type:  model.BlockContentTextMark_Mention,
+					Param: fileNameToObjectId[testMdPath],
+				},
+				{
+					Range: &model.Range{From: 33, To: 38},
+					Type:  model.BlockContentTextMark_Bold,
+				},
+			}})),
+			blockbuilder.Text("Test file block with bold mark test3", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 31, To: 36},
+					Type:  model.BlockContentTextMark_Link,
+					Param: testTxtPath,
+				},
+				{
+					Range: &model.Range{From: 31, To: 36},
+					Type:  model.BlockContentTextMark_Bold,
+				},
+			}})),
+			blockbuilder.Text("Test link to csv with bold mark test4", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 32, To: 37},
+					Type:  model.BlockContentTextMark_Mention,
+					Param: fileNameToObjectId[testCsvPath],
+				},
+				{
+					Range: &model.Range{From: 32, To: 37},
+					Type:  model.BlockContentTextMark_Bold,
+				},
+			}})),
+			blockbuilder.Bookmark(fileMdPath),
+			blockbuilder.Text("test2", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 0, To: 5},
+					Type:  model.BlockContentTextMark_Mention,
+					Param: fileNameToObjectId[testMdPath],
+				},
+				{
+					Range: &model.Range{From: 0, To: 5},
+					Type:  model.BlockContentTextMark_Bold,
+				},
+			}})),
+			blockbuilder.File("", blockbuilder.FileName(provider.TempDir()+testTxtPath), blockbuilder.FileType(model.BlockContentFile_File)),
+			blockbuilder.Text("test4", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
+				{
+					Range: &model.Range{From: 0, To: 5},
+					Type:  model.BlockContentTextMark_Mention,
+					Param: fileNameToObjectId[testCsvPath],
+				},
+				{
+					Range: &model.Range{From: 0, To: 5},
+					Type:  model.BlockContentTextMark_Bold,
+				},
+			}})),
+			blockbuilder.Link(rootId),
+		))
+	return want
 }
 
 func setupTestDirectory(t *testing.T) string {

core/block/import/markdown/testdata/links.md

@@ -0,0 +1,23 @@
+File does not exist [test1](file.md)
+
+Test link to page [test2](test.md)
+
+Test file block [test3](test.txt)
+
+Test link to csv [test4](test.csv)
+
+File does not exist with bold mark **[test1](file.md)**
+
+Test link to page with bold mark **[test2](test.md)**
+
+Test file block with bold mark **[test3](test.txt)**
+
+Test link to csv with bold mark **[test4](test.csv)**
+
+**[test1](file.md)**
+
+**[test2](test.md)**
+
+**[test3](test.txt)**
+
+**[test4](test.csv)**