From df786f2aaffd5709dd3bc0bd6ea35f8a0f573887 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Wed, 8 Jan 2025 12:41:41 +0300
Subject: [PATCH] add support for comments in files (#224)

* add support for comments in files

* update lint action

* fix lint errs
---
 .github/workflows/lint-test.yml | 18 +++++-------------
 ratelimit_var.go                |  2 +-
 runtime_map.go                  |  2 +-
 slice_common.go                 | 31 +++++++++++++++++++++++++------
 string_slice_test.go            |  2 +-
 5 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
index c5b9f6c..e387cca 100644
--- a/.github/workflows/lint-test.yml
+++ b/.github/workflows/lint-test.yml
@@ -9,18 +9,10 @@ on:
 
 jobs:
   lint:
-    name: Lint Test
+    name: "Lint"
+    if: "${{ !endsWith(github.actor, '[bot]') }}"
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.21.x
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.0
-        with:
-          version: latest
-          args: --timeout 5m
-          working-directory: .
+      - uses: actions/checkout@v4
+      - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/actions/golangci-lint@v1
diff --git a/ratelimit_var.go b/ratelimit_var.go
index bcd9cdb..e02ad51 100644
--- a/ratelimit_var.go
+++ b/ratelimit_var.go
@@ -77,7 +77,7 @@ func (rateLimitMap *RateLimitMap) Del(key string) error {
 
 // IsEmpty specifies if the underlying map is empty
 func (rateLimitMap *RateLimitMap) IsEmpty() bool {
-	return rateLimitMap.kv == nil || len(rateLimitMap.kv) == 0
+	return len(rateLimitMap.kv) == 0
 }
 
 // AsMap returns the internal map as reference - changes are allowed
diff --git a/runtime_map.go b/runtime_map.go
index f0af3e7..7958a2b 100644
--- a/runtime_map.go
+++ b/runtime_map.go
@@ -60,7 +60,7 @@ func (runtimeMap *RuntimeMap) Del(key string) error {
 
 // IsEmpty specifies if the underlying map is empty
 func (runtimeMap *RuntimeMap) IsEmpty() bool {
-	return runtimeMap.kv == nil || len(runtimeMap.kv) == 0
+	return len(runtimeMap.kv) == 0
 }
 
 // AsMap returns the internal map as reference - changes are allowed
diff --git a/slice_common.go b/slice_common.go
index 720b429..4805482 100644
--- a/slice_common.go
+++ b/slice_common.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/pkg/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 var quotes = []rune{'"', '\'', '`'}
@@ -64,10 +65,10 @@ func ToStringSlice(value string, options Options) ([]string, error) {
 	}
 
 	addPartToResult := func(part string) {
+		if options.Normalize != nil {
+			part = options.Normalize(part)
+		}
 		if !options.IsEmpty(part) {
-			if options.Normalize != nil {
-				part = options.Normalize(part)
-			}
 			result = append(result, part)
 		}
 	}
@@ -120,13 +121,31 @@ func isFromFile(_ string) bool {
 }
 
 func normalizeTrailingParts(s string) string {
-	return strings.TrimSpace(s)
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimSpaces:    true,
+		},
+	)
 }
 
 func normalize(s string) string {
-	return strings.TrimSpace(strings.Trim(strings.TrimSpace(s), string(quotes)))
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimCutset:    string(quotes),
+			TrimSpaces:    true,
+		},
+	)
 }
 
 func normalizeLowercase(s string) string {
-	return strings.TrimSpace(strings.Trim(strings.TrimSpace(strings.ToLower(s)), string(quotes)))
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimCutset:    string(quotes),
+			TrimSpaces:    true,
+			Lowercase:     true,
+		},
+	)
 }
diff --git a/string_slice_test.go b/string_slice_test.go
index 023ab7d..447f3ae 100644
--- a/string_slice_test.go
+++ b/string_slice_test.go
@@ -86,7 +86,7 @@ func TestFileNormalizedStringSliceOptions(t *testing.T) {
 
 func TestFileStringSliceOptions(t *testing.T) {
 	filename := "test.txt"
-	_ = os.WriteFile(filename, []byte("value1,value2\nvalue3"), 0644)
+	_ = os.WriteFile(filename, []byte("# this is a comment\nvalue1,value2\nvalue3"), 0644)
 	defer os.RemoveAll(filename)
 
 	result, err := ToStringSlice(filename, FileStringSliceOptions)