Merge pull request #317 from pzaino/develop

Improved human-like behavior and added more SSL analysis

README.md

@@ -254,6 +254,11 @@ Raspberry Pi.
 **Please Note(3)**: If need to do a rebuild and want to clean up everything,
 run the following command:
 
+**Please Note(4)**: To build the CROWler VDI docker image, it's required to
+build also Selenium (don't worry everything is automatic), however you need
+to ensure that GNU Make is installed on your system. That is required to
+build selenium images (nothing to do with the CROWler itself).
+
 ```bash
 ./docker-rebuild.sh
 ```

go.mod

@@ -23,6 +23,7 @@ require (
 require (
 	github.com/Ullaakut/nmap/v3 v3.0.3
 	github.com/jmoiron/sqlx v1.4.0
+	github.com/spaolacci/murmur3 v1.1.0
 	golang.org/x/crypto v0.24.0
 )
 

go.sum

@@ -95,6 +95,8 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
@@ -169,6 +171,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/common/enconding.go

@@ -0,0 +1,11 @@
+package common
+
+import (
+	"encoding/base64"
+)
+
+// Base64Encode encodes a string to base64, this may be required by some
+// configurations.
+func Base64Encode(data string) string {
+	return base64.StdEncoding.EncodeToString([]byte(data))
+}

pkg/config/config.go

@@ -253,9 +253,25 @@ func NewConfig() *Config {
 			SSLMode: "disable",
 		},
 		HTTPHeaders: HTTPConfig{
-			Enabled:      true,
-			Timeout:      60,
-			SSLDiscovery: true,
+			Enabled: true,
+			Timeout: 60,
+			SSLDiscovery: SSLScoutConfig{
+				Enabled:     true,
+				JARM:        false,
+				JA3:         false,
+				JA3S:        true,
+				HASSH:       false,
+				HASSHServer: true,
+				TLSH:        true,
+				SimHash:     true,
+				MinHash:     true,
+				BLAKE2:      true,
+				SHA256:      true,
+				CityHash:    true,
+				MurmurHash:  true,
+				CustomTLS:   true,
+			},
+			Proxies: []SOCKSProxy{},
 		},
 		NetworkInfo: NetworkInfo{
 			DNS: DNSConfig{
@@ -1014,7 +1030,7 @@ func IsEmpty(config Config) bool {
 		return false
 	}
 
-	if config.HTTPHeaders != (HTTPConfig{}) {
+	if !config.HTTPHeaders.IsEmpty() {
 		return false
 	}
 
@@ -1063,7 +1079,7 @@ func (ssc *ServiceScoutConfig) IsEmpty() bool {
 	return true
 }
 
-// isEmpty checks if the given ExecutionPlanItem is empty.
+// IsEmpty checks if the given ExecutionPlanItem is empty.
 func (ep *ExecutionPlanItem) IsEmpty() bool {
 	if ep == nil {
 		return true
@@ -1076,7 +1092,7 @@ func (ep *ExecutionPlanItem) IsEmpty() bool {
 	return false
 }
 
-// isEmpty checks if the given SourceConfig is empty.
+// IsEmpty checks if the given SourceConfig is empty.
 func (sc *SourceConfig) IsEmpty() bool {
 	if sc == nil {
 		return true
@@ -1089,6 +1105,7 @@ func (sc *SourceConfig) IsEmpty() bool {
 	return true
 }
 
+// IsEmpty checks if the given Config is empty.
 func (cfg *Config) IsEmpty() bool {
 	if cfg == nil {
 		return true
@@ -1122,7 +1139,7 @@ func (cfg *Config) IsEmpty() bool {
 		return false
 	}
 
-	if cfg.HTTPHeaders != (HTTPConfig{}) {
+	if !cfg.HTTPHeaders.IsEmpty() {
 		return false
 	}
 
@@ -1144,3 +1161,112 @@ func (cfg *Config) IsEmpty() bool {
 
 	return true
 }
+
+// IsEmpty checks if the given DNSConfig is empty.
+func (dc *DNSConfig) IsEmpty() bool {
+	if dc == nil {
+		return true
+	}
+
+	if dc.Enabled {
+		return false
+	}
+
+	if dc.Timeout != 0 {
+		return false
+	}
+
+	if dc.RateLimit != "" {
+		return false
+	}
+
+	return true
+}
+
+// IsEmpty checks if the given WHOISConfig is empty.
+func (wc *WHOISConfig) IsEmpty() bool {
+	if wc == nil {
+		return true
+	}
+
+	if wc.Enabled {
+		return false
+	}
+
+	if wc.Timeout != 0 {
+		return false
+	}
+
+	if wc.RateLimit != "" {
+		return false
+	}
+
+	return true
+}
+
+// IsEmpty checks if the given NetLookupConfig is empty.
+func (nlc *NetLookupConfig) IsEmpty() bool {
+	if nlc == nil {
+		return true
+	}
+
+	if nlc.Enabled {
+		return false
+	}
+
+	if nlc.Timeout != 0 {
+		return false
+	}
+
+	if nlc.RateLimit != "" {
+		return false
+	}
+
+	return true
+}
+
+// IsEmpty checks if the given GeoLookupConfig is empty.
+func (glc *GeoLookupConfig) IsEmpty() bool {
+	if glc == nil {
+		return true
+	}
+
+	if glc.Enabled {
+		return false
+	}
+
+	if glc.Type != "" {
+		return false
+	}
+
+	if glc.DBPath != "" {
+		return false
+	}
+
+	return true
+}
+
+// IsEmpty checks if the given HTTPConfig is empty.
+func (hc *HTTPConfig) IsEmpty() bool {
+	if hc == nil {
+		return true
+	}
+
+	if hc.Enabled {
+		return false
+	}
+
+	if hc.Timeout != 0 {
+		return false
+	}
+
+	if hc.SSLDiscovery != (SSLScoutConfig{}) {
+		return false
+	}
+
+	if len(hc.Proxies) != 0 {
+		return false
+	}
+
+	return true
+}

pkg/config/config_test.go

@@ -1204,7 +1204,7 @@ func TestConfigString(t *testing.T) {
 	}
 
 	// Define the expected string representation of the config
-	expected := "Config{Remote: {https://example.com /api 8080 us-west-1 mytoken  0  }, Database: {  0 testuser testpassword  0 0  }, Crawler: {0  0 0 false false 0 0 0 0   0 0 false false false false false false 0 false}, API: { 0 0 false false     false 0 0 0 false}, Selenium: [{    chrome  4444  false false    }], RulesetsSchemaPath: path/to/schema, Rulesets: [], ImageStorageAPI: {  0    0  }, FileStorageAPI: {  0    0  }, HTTPHeaders: {false 0 false false}, NetworkInfo: {{false 0 } {false 0 } {false 0 } {false 0 { 0} false false false false false false  false false [] [] []    0 0 0   false 0  false  false 0 [] []} {false    0 } {  }}, OS: linux, DebugLevel: 1}"
+	expected := "Config{Remote: {https://example.com /api 8080 us-west-1 mytoken  0  }, Database: {  0 testuser testpassword  0 0  }, Crawler: {0  0 0 false false 0 0 0 0   0 0 false false false false false false 0 false}, API: { 0 0 false false     false 0 0 0 false}, Selenium: [{    chrome  4444  false false    }], RulesetsSchemaPath: path/to/schema, Rulesets: [], ImageStorageAPI: {  0    0  }, FileStorageAPI: {  0    0  }, HTTPHeaders: {false 0 false {false false false false false false false false false false false false false false} []}, NetworkInfo: {{false 0 } {false 0 } {false 0 } {false 0 { 0} false false false false false false  false false [] [] []    0 0 0   false 0  false  false 0 [] []} {false    0 } {  }}, OS: linux, DebugLevel: 1}"
 
 	// Call the String method on the config
 	result := config.String()

pkg/config/types.go

@@ -102,10 +102,34 @@ type GeoLookupConfig struct {
 }
 
 type HTTPConfig struct {
-	Enabled         bool `yaml:"enabled"`
-	Timeout         int  `yaml:"timeout"`
-	FollowRedirects bool `yaml:"follow_redirects"`
-	SSLDiscovery    bool `yaml:"ssl_discovery"`
+	Enabled         bool           `yaml:"enabled"`
+	Timeout         int            `yaml:"timeout"`
+	FollowRedirects bool           `yaml:"follow_redirects"`
+	SSLDiscovery    SSLScoutConfig `yaml:"ssl_discovery"`
+	Proxies         []SOCKSProxy   `yaml:"proxies"`
+}
+
+type SSLScoutConfig struct {
+	Enabled     bool `yaml:"enabled"`
+	JARM        bool `yaml:"jarm"`
+	JA3         bool `yaml:"ja3"`
+	JA3S        bool `yaml:"ja3s"`
+	HASSH       bool `yaml:"hassh"`
+	HASSHServer bool `yaml:"hassh_server"`
+	TLSH        bool `yaml:"tlsh"`
+	SimHash     bool `yaml:"simhash"`
+	MinHash     bool `yaml:"minhash"`
+	BLAKE2      bool `yaml:"blake2"`
+	SHA256      bool `yaml:"sha256"`
+	CityHash    bool `yaml:"cityhash"`
+	MurmurHash  bool `yaml:"murmurhash"`
+	CustomTLS   bool `yaml:"custom_tls"`
+}
+
+type SOCKSProxy struct {
+	Address  string
+	Username string
+	Password string
 }
 
 // ServiceScoutConfig represents a structured configuration for an Nmap scan.