Add cache

configuration/linkcheck.json

@@ -1,5 +1,7 @@
 {
   "exclude_links": [],
   "only_errors": true,
-  "serial": false
+  "serial": false,
+  "cache_duration": "24h",
+  "cache_output_path": "cache_store/cache"
 }

main.go

@@ -17,6 +17,7 @@ import (
 // TODO fix readme
 // TODO mock external server behavior
 func main() {
+	log.Info("Starting linkcheck")
 	start := time.Now()
 	log.SetLevel(log.InfoLevel)
 	_, b, _, _ := runtime.Caller(0)

mega-linter-plugin-linkcheck/linkcheck.megalinter-descriptor.yml

@@ -16,7 +16,8 @@ linters:
     cli_version_arg_name: "-version"
     cli_lint_fix_arg_name: "--megalinter-fix-flag" # Workaround for MegaLinter
     cli_lint_mode: list_of_files
-    lint_all_files: true
+    lint_all_files: ${LINT_ALL_FILES:-true}  # Use default value if LINT_ALL_FILES is not set
+    git_diff: "${GIT_DIFF_COMMAND}"
     cli_lint_errors_count: regex_sum
     cli_lint_errors_regex: "ERROR: ([0-9]+) links check failed, please check the logs"
     examples:
@@ -26,4 +27,4 @@ linters:
       - "linkcheck --config linkcheck.json"
     install:
       dockerfile:
-        - RUN export GO111MODULE=on && go install github.com/shiranr/linkcheck@v1.1.4 && go clean --cache
+        - RUN export GO111MODULE=on && go install github.com/shiranr/linkcheck@v2.0.17.beta && go clean --cache

models/files_processor.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"sync"
 )
@@ -32,6 +33,7 @@ func GetFilesProcessorInstance() FilesProcessor {
 
 // Process - process the multiple files list
 func (fh *filesProcessor) Process(files []string) error {
+	log.Info("Starting to process links")
 	for _, filePath := range files {
 		fileLinkData := FileResultData{
 			FilePath: filePath,
@@ -47,7 +49,10 @@ func (fh *filesProcessor) Process(files []string) error {
 	}
 	wg.Wait()
 	fh.Close()
-	return fh.Print()
+	result := fh.Print()
+	cache := GetCacheInstance(false)
+	cache.Close()
+	return result
 }
 
 func (fh *filesProcessor) invoke(fileProcessor FileProcessor) {

models/link_cache.go

@@ -0,0 +1,149 @@
+package models
+
+import (
+	"encoding/gob"
+	"errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+var cache *LinksCache
+var filePath string
+
+type LinksCache struct {
+	linksCache map[string]*LinkData
+	mapLock    sync.RWMutex
+	duration   time.Duration
+	runNumber  uint64
+}
+
+type LinkData struct {
+	ResponseStatus int
+	LastChecked    int64
+	LinkPath       string
+	RunNumber      uint64
+}
+
+// Please notice this is not thread safe
+func GetCacheInstance(empty bool) *LinksCache {
+	if cache == nil {
+		filePath = viper.GetString("cache_output_path")
+		duration := viper.GetDuration("cache_duration")
+		cache = &LinksCache{
+			linksCache: make(map[string]*LinkData),
+			mapLock:    sync.RWMutex{},
+			duration:   duration,
+			runNumber:  rand.Uint64(),
+		}
+		if !empty {
+			log.Info("Loading cache data from " + filePath)
+			cache.loadCacheData()
+		}
+	}
+	return cache
+}
+
+func (c *LinksCache) Close() {
+	c.SaveCache()
+	cache = nil
+}
+
+func (c *LinksCache) loadCacheData() {
+	if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
+		log.Error("File "+filePath+" does not exist, skipping cache load", err)
+		return
+	}
+	log.Info("Opening cache file " + filePath)
+	file, err := os.Open(filePath)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+	c.decodeData(file)
+}
+
+func (c *LinksCache) SaveCache() {
+	basePath := filepath.Dir(filePath)
+	log.Info("Saving cache to path " + basePath)
+	file, err := os.Create(filePath)
+	if err != nil {
+		log.Error("Failed to save cache to  "+basePath, err)
+		return
+	}
+	defer file.Close()
+
+	c.encodeData(file)
+}
+
+func (c *LinksCache) AddLink(linkPath string, status int) {
+	c.mapLock.Lock()
+	defer c.mapLock.Unlock()
+	data := c.linksCache[linkPath]
+	if data == nil {
+		data = &LinkData{
+			ResponseStatus: status,
+			LastChecked:    time.Now().Unix(),
+			LinkPath:       linkPath,
+			RunNumber:      c.runNumber,
+		}
+	} else {
+		data.ResponseStatus = status
+		data.RunNumber = c.runNumber
+		data.LastChecked = time.Now().Unix()
+	}
+	c.linksCache[linkPath] = data
+}
+
+func (c *LinksCache) IsTimeCachedElapsed(linkPath string) bool {
+	c.mapLock.RLock()
+	defer c.mapLock.RUnlock()
+	val, ok := c.linksCache[linkPath]
+	if !ok {
+		return true
+	}
+	return c.checkTimeElapsed(val)
+}
+
+func (c *LinksCache) checkTimeElapsed(val *LinkData) bool {
+	if val.LastChecked+int64(c.duration.Seconds()) < time.Now().Unix() {
+		return true
+	}
+	if val.RunNumber != c.runNumber && (299 < val.ResponseStatus || val.ResponseStatus < 200) {
+		return true
+	}
+	return false
+}
+
+func (c *LinksCache) CheckLinkStatus(linkPath string) (int, bool) {
+	c.mapLock.RLock()
+	defer c.mapLock.RUnlock()
+	val, ok := c.linksCache[linkPath]
+	if !ok {
+		return 0, ok
+	}
+	if c.checkTimeElapsed(val) {
+		return 0, false
+	}
+	return val.ResponseStatus, ok
+}
+
+func (c *LinksCache) encodeData(file *os.File) {
+	encoder := gob.NewEncoder(file)
+	err := encoder.Encode(c.linksCache)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (c *LinksCache) decodeData(file *os.File) {
+	decoder := gob.NewDecoder(file)
+	err := decoder.Decode(&c.linksCache)
+	if err != nil {
+		panic(err)
+	}
+}

models/link_processor.go

@@ -4,7 +4,6 @@ import (
 	"github.com/spf13/viper"
 	"regexp"
 	"strings"
-	"sync"
 )
 
 var link = "\\[{1}([é’*&\"|`?'>\\-\\sa-zA-Z0-9@:%._\\\\+~#=,\\n\\/\\(\\)])*((\\]\\()){1}([?\\sa-zA-Z0-9@:%._\\\\+~#=\\/\\/\\-]{1,256}(\\(.*\\))?(\\\"(.*)\\\")?)\\){1}"
@@ -18,29 +17,11 @@ type LinkProcessor interface {
 }
 
 type linkProcessor struct {
-	linksCache
+	LinksCache
 	mdLinkRegex   *regexp.Regexp
 	excludedLinks []string
 }
 
-type linksCache struct {
-	linksCache map[string]int
-	mapLock    sync.RWMutex
-}
-
-func (cache *linksCache) addLink(linkPath string, status int) {
-	cache.mapLock.Lock()
-	defer cache.mapLock.Unlock()
-	cache.linksCache[linkPath] = status
-}
-
-func (cache *linksCache) checkLinkCache(linkPath string) (int, bool) {
-	cache.mapLock.RLock()
-	defer cache.mapLock.RUnlock()
-	val, ok := cache.linksCache[linkPath]
-	return val, ok
-}
-
 var lh *linkProcessor
 
 // GetLinkProcessorInstance - get instance of link processor
@@ -49,11 +30,9 @@ func GetLinkProcessorInstance() LinkProcessor {
 	regex, _ := regexp.Compile(linkOrPath)
 	if lh == nil {
 		lh = &linkProcessor{
-			linksCache{
-				linksCache: map[string]int{},
-			},
-			regex,
-			viper.GetStringSlice("exclude_links"),
+			LinksCache:    *GetCacheInstance(false),
+			mdLinkRegex:   regex,
+			excludedLinks: viper.GetStringSlice("exclude_links"),
 		}
 	}
 	return lh
@@ -67,7 +46,7 @@ func (processor *linkProcessor) CheckLink(filePath string, linkPath string, line
 		path:       linkPath,
 		filePath:   filePath,
 	}
-	status, ok := processor.checkLinkCache(linkPath)
+	status, ok := processor.CheckLinkStatus(linkPath)
 	if !ok {
 		switch {
 		case strings.HasPrefix(linkData.path, "http"):
@@ -83,7 +62,7 @@ func (processor *linkProcessor) CheckLink(filePath string, linkPath string, line
 			fileLinkHandler := GetInternalLinkHandler(filePath)
 			linkData.status = fileLinkHandler.Handle(linkPath)
 		}
-		processor.addLink(linkPath, linkData.status)
+		processor.AddLink(linkPath, linkData.status)
 	} else {
 		linkData.status = status
 	}

tests/integration_test.go

@@ -8,28 +8,34 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"path/filepath"
-	"runtime"
 	"testing"
 )
 
 func TestFilesWithError(t *testing.T) {
-	_, b, _, _ := runtime.Caller(0)
-	logFilePath := setLogFile()
-	basepath := filepath.Dir(b)
-	configPath := basepath + "/resources/linkcheck.json"
-	utils.LoadConfiguration(configPath)
+	cache := models.GetCacheInstance(true)
+	validateLinkInCache(t, cache, "/~https://github.com/apache/jmeter", 0, false)
 	readmeFiles := utils.ExtractMarkdownFiles()
 	res := models.GetFilesProcessorInstance().Process(readmeFiles)
 	assert.ErrorContains(t, res, "ERROR: 3 links check failed, please check the logs")
+	logFilePath := setLogFile()
 	logFileContent := readLogFile(logFilePath)
 	assert.Contains(t, logFileContent, "Went through 2 files")
 	assert.Contains(t, logFileContent, "Line 21 link nla.go status 400")
 	assert.Contains(t, logFileContent, "Line 33 link source-control/merge-strategies.md status 400")
 	assert.Contains(t, logFileContent, "Line 33 link resources/templates/CONTRIBUTING.md status 400")
 	assert.NotContains(t, logFileContent, "http://bla.com/")
 	assert.NotContains(t, logFileContent, "http://test.com/")
+	cache = models.GetCacheInstance(false)
+	validateLinkInCache(t, cache, "http://bla.com/", 0, false)
+	validateLinkInCache(t, cache, "http://test.com/", 0, false)
+	validateLinkInCache(t, cache, "/~https://github.com/apache/jmeter", 200, true)
+
+}
 
+func validateLinkInCache(t *testing.T, cache *models.LinksCache, linkPath string, expectedStatus int, isOk bool) {
+	status, ok := cache.CheckLinkStatus(linkPath)
+	assert.Equal(t, ok, isOk)
+	assert.Equal(t, status, expectedStatus)
 }
 
 func readLogFile(filePath string) string {

tests/link_cache_test.go

@@ -0,0 +1,47 @@
+package tests
+
+import (
+	"github.com/shiranr/linkcheck/models"
+	"github.com/shiranr/linkcheck/utils"
+	"github.com/stretchr/testify/assert"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+var cache *models.LinksCache
+
+func init() {
+	_, b, _, _ := runtime.Caller(0)
+	basepath := filepath.Dir(b)
+	configPath := basepath + "/resources/linkcheck.json"
+	utils.LoadConfiguration(configPath)
+	cache = models.GetCacheInstance(true)
+	cache.SaveCache()
+}
+
+func TestInstancesAreTheSame(t *testing.T) {
+	cache2 := models.GetCacheInstance(false)
+	assert.Equal(t, cache, cache2)
+}
+
+func TestCacheIsNotNil(t *testing.T) {
+	assert.NotNil(t, cache)
+}
+
+func TestAddingDataToCache(t *testing.T) {
+	respStat, ok := cache.CheckLinkStatus("test")
+	assert.Equal(t, respStat, 0)
+	assert.False(t, ok)
+	cache.AddLink("test", 200)
+	cache.Close()
+	cache = models.GetCacheInstance(false)
+	respStat, ok = cache.CheckLinkStatus("test")
+	assert.Equal(t, respStat, 200)
+	assert.True(t, ok)
+}
+
+func TestCacheIsNotNilAfterClose(t *testing.T) {
+	cache.Close()
+	assert.NotNil(t, cache)
+}

tests/resources/linkcheck.json

@@ -3,5 +3,7 @@
     "https://test.com",
     "http://bla.com/"
   ],
-  "only_errors": false
+  "only_errors": false,
+  "cache_duration": "24h",
+  "cache_output_path": "resources/test_cache"
 }