merge

.gitignore

@@ -3,3 +3,5 @@ git-hound
 .goreleaser.yaml
 
 dist/
+.venv
+output*

README.md

@@ -43,7 +43,7 @@ GitHound utilizes a database of API key regexes maintained by the [Gitleaks](htt
 
 Knowing the pattern for a specific service's API keys enables you to search GitHub for these keys. You can then pipe matches for your custom key regex into your own script to test the API key against the service and to identify the at-risk account.
 
-`echo "api.halcorp.biz" | githound --dig-files --dig-commits --many-results --regex-file halcorp-api-regexes.txt --results-only | python halapitester.py`
+`echo "api.halcorp.biz" | githound --dig-files --dig-commits --many-results --rules halcorp-api-regexes.txt --results-only | python halapitester.py`
 
 For detecting future API key leaks, GitHub offers [Push Token Scanning](https://help.github.com/en/articles/about-token-scanning) to immediately detect API keys as they are posted.
 
@@ -62,34 +62,43 @@ For files that encode secrets, decodes base64 strings and searches the encoded s
 Check out this [blog post](https://tillsongalloway.com/finding-sensitive-information-on-github/) for more details on use cases and methodologies.
 
 ## Flags
+GitHound makes it easy to find exposed API keys on GitHub using pattern matching, targetted querying, and a robust scoring system.
 ```
 Usage:
   githound [flags]
 
 Flags:
-      --config-file string      Supply the path to a config file.
-      --debug                   Enables verbose debug logging.
-      --dig-commits             Dig through commit history to find more secrets (CPU intensive).
-      --dig-files               Dig through the repo's files to find more secrets (CPU intensive).
-      --filtered-only           Only print filtered results (language files)
-      --github-repo             Search in a specific Github Repo only.
-  -h, --help                    help for githound
-      --json                    Print results in JSON format
-      --language-file string    Supply your own list of languages to search (java, python).
-      --legacy                  Use the legacy search method.
-      --many-results            Search >100 pages with filtering hack
-      --no-api-keys             Don't search for generic API keys.
-      --no-files                Don't search for interesting files.
-      --no-gists                Don't search Gists
-      --no-keywords             Don't search for built-in keywords
-      --no-repos                Don't search repos
-      --no-scoring              Don't use scoring to filter out false positives.
-      --otp-code string         Github account 2FA token used for sign-in. (Only use if you have 2FA enabled on your account via authenticator app)
-      --pages int               Maximum pages to search per query (default 100)
-      --regex-file string       Path to a list of regexes. (default "rules.toml")
-      --results-only            Only print match strings.
-      --subdomain-file string   A file containing a list of subdomains (or other queries).
-      --threads int             Threads to dig with (default 20)
+      --query string         A query stiing (alternativ, pass through stdin)
+      --config-file string   Supply the path to a config file.
+      --json                 Print results in JSON format
+      --rules string         Path to a list of regexes or a GitLeaks rules folder.
+      -h, --help                 help for githound
+
+      --pages int            Maximum pages to search per query (default 100)
+      --github-repo          Search in a specific Github Repo only.
+      --fast                 Skip file grepping and only return search preview
+      --all-results          Print all results, even if they do not contain secrets
+      --many-results         Search >100 pages with results sorting hack
+
+      --dig-commits          Dig through commit history to find more secrets (CPU intensive).
+      --dig-files            Dig through the repo's files to find more secrets (CPU intensive).
+      --threads int          Threads to dig with (default 20)```
+
+      --results-only         Only print match strings.
+      --search-type api      Search interface (api or `ui`). API requires api_key in config, UI requires username/password
+
+      --otp-code string      Github account 2FA token used for sign-in. (Only use if you have 2FA enabled on your account via authenticator app)
+
+      --query-file string    A file containing a list of subdomains (or other queries).
+      --filtered-only        Only print filtered results (language files)
+      --debug                Enables verbose debug logging.
+  
+      --no-api-keys          Don't search for generic API keys.
+      --no-files             Don't search for interesting files.
+      --no-gists             Don't search Gists
+      --no-keywords          Don't search for built-in keywords
+      --no-repos             Don't search repos
+      --no-scoring           Don't use scoring to filter out false positives.
 ```
 
 ## Development
@@ -146,23 +155,10 @@ docker run -v /path/to/config.yaml:/root/.githound/config.yaml -v $(pwd)/data:/d
 
 Replace `/path/to/config.yaml` with the actual path to your `config.yaml` file. The `-v $(pwd)/data:/data` part mounts a directory containing your input files (`subdomains.txt`) into the container.
 
-#### Notes
-- Ensure your `config.yaml` and input files' paths are correct when running the Docker container.
-- This setup assumes `git-hound` is compatible with the provided configuration and command-line arguments.
-- For any updates or changes to `git-hound`, rebuild the Docker image.
-
 ---
 
-
-## User feedback
-
-These are discussions about how people use GitHound in their workflows and how we can GitHound to fufill those needs. If you use GitHound, consider leaving a note in one of the active issues.
-[List of issues requesting user feedback](/~https://github.com/tillson/git-hound/issues?q=is%3Aissue+is%3Aopen+label%3A%22user+feedback+requested%22)
-
-## 💰 Premium Monitoring & Engagements
-Would you like to gain greater visibility into your company's GitHub presence? We use GitHound as one small part of a larger system that can find credential leaks and sensitive/proprietary information across open-source websites like GitHub and DockerHub. We offer continuous monitoring services of *all of GitHub* (not just accounts you know are held by employees!) and red-team engagements/consulting services.
-
-Reach out here to learn more: https://secretsurfer.xyz.
+## Premium Support Options
+Bug fixes and occasional feature enhancements are provided open-source. Technical support and integration requests are available at a rate of 35 USD/hour and can be arranged by contacting tillson@secretsurfer.xyz.
 
 ## References
 

cmd/root.go

@@ -3,34 +3,41 @@ package cmd
 import (
 	"bufio"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 
-	"github.com/BurntSushi/toml"
+	"github.com/GRbit/go-pcre"
 	"github.com/fatih/color"
 	"github.com/spf13/viper"
 	"golang.org/x/crypto/ssh/terminal"
+	"gopkg.in/yaml.v2"
 
 	"strings"
 
+	"github.com/BurntSushi/toml"
 	"github.com/spf13/cobra"
 	"github.com/tillson/git-hound/internal/app"
+
+	_ "net/http/pprof"
 )
 
 // InitializeFlags initializes GitHound's command line flags.
 func InitializeFlags() {
+	rootCmd.PersistentFlags().StringVar(&app.GetFlags().SearchType, "search-type", "", "Search interface (`api` or `ui`).")
 	rootCmd.PersistentFlags().StringVar(&app.GetFlags().QueryFile, "query-file", "", "A file containing a list of subdomains (or other queries).")
 	rootCmd.PersistentFlags().StringVar(&app.GetFlags().Query, "query", "", "A query stiing (default: stdin)")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().DigRepo, "dig-files", false, "Dig through the repo's files to find more secrets (CPU intensive).")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().DigCommits, "dig-commits", false, "Dig through commit history to find more secrets (CPU intensive).")
-	rootCmd.PersistentFlags().StringVar(&app.GetFlags().RegexFile, "regex-file", "rules.toml", "Path to a list of regexes.")
-	rootCmd.PersistentFlags().StringVar(&app.GetFlags().LanguageFile, "language-file", "", "Supply your own list of languages to search (java, python).")
+	rootCmd.PersistentFlags().StringVar(&app.GetFlags().RegexFile, "rules", "rules/rules-noseyparker", "Path to a list of regexes or a GitLeaks rules folder.")
+	rootCmd.PersistentFlags().StringVar(&app.GetFlags().RegexFile, "regex-file", "rules/rules-noseyparker", "Alias for the 'rules' flag.")
+	rootCmd.PersistentFlags().MarkHidden("regex-file")
 	rootCmd.PersistentFlags().StringVar(&app.GetFlags().ConfigFile, "config-file", "", "Supply the path to a config file.")
 	rootCmd.PersistentFlags().IntVar(&app.GetFlags().Pages, "pages", 100, "Maximum pages to search per query")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().GithubRepo, "github-repo", false, "Search in a specific Github Repo only.")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().ResultsOnly, "results-only", false, "Only print match strings.")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoAPIKeys, "no-api-keys", false, "Don't search for generic API keys.")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoScoring, "no-scoring", false, "Don't use scoring to filter out false positives.")
-	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().LegacySearch, "legacy", false, "Use the legacy search method.")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoFiles, "no-files", false, "Don't search for interesting files.")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoKeywords, "no-keywords", false, "Don't search for built-in keywords")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().ManyResults, "many-results", false, "Search >100 pages with filtering hack")
@@ -79,45 +86,125 @@ var rootCmd = &cobra.Command{
 			}
 		}
 		if len(queries) == 0 {
-			color.Red("[!] No search queries specified.")
+			color.Red("[!] No search queries specified. Use flag `--query [query]`, or pipe query into GitHound.")
 			os.Exit(1)
 			return
 		}
 
-		client, err := app.LoginToGitHub(app.GitHubCredentials{
-			Username: viper.GetString("github_username"),
-			Password: viper.GetString("github_password"),
-			OTP:      viper.GetString("github_totp_seed"),
-		})
-
-		if err != nil {
-			fmt.Println(err)
-			color.Red("[!] Unable to login to GitHub.")
-			os.Exit(1)
-		}
-		if !app.GetFlags().ResultsOnly && !app.GetFlags().JsonOutput {
-			color.Cyan("[*] Logged into GitHub as " + viper.GetString("github_username"))
-		}
-		toml.DecodeFile(app.GetFlags().RegexFile, &app.GetFlags().TextRegexes)
-		for _, query := range queries {
-			_, err = app.Search(query, client)
+		var allRules []app.Rule
+		// fmt.Println(app.GetFlags().RegexFile)
+		// If rules is a directory, load all rules files in GitLeaks YML format
+		if fileInfo, err := os.Stat(app.GetFlags().RegexFile); err == nil && fileInfo.IsDir() {
+			files, err := ioutil.ReadDir(app.GetFlags().RegexFile)
 			if err != nil {
-				color.Red("[!] Unable to collect search results for query '" + query + "'.")
-				break
+				fmt.Println(err)
+				os.Exit(1)
+			}
+			for _, file := range files {
+				// if filepath.Ext(file.Name()) == ".yml" || filepath.Ext(file.Name()) == ".yml" {
+				filePath := filepath.Join(app.GetFlags().RegexFile, file.Name())
+				rules := LoadRegexFile(filePath)
+				allRules = append(allRules, rules...)
+				// }
 			}
+			app.GetFlags().TextRegexes = append(app.GetFlags().TextRegexes, allRules...)
+		} else {
+			// Otherwise, resort to regex list in txt file or legacy TOML files
+			rules := LoadRegexFile(app.GetFlags().RegexFile)
+			allRules = append(allRules, rules...)
 		}
-		size, err = app.DirSize("/tmp/githound")
-		if err == nil && size > 50e+6 {
-			app.ClearRepoStorage()
+		if len(allRules) == 0 {
+			color.Yellow("[!] 0 rules loaded. Using an empty ruleset may result in lousy performance. Consider using one of the rulesets provided with the GitHound installation or available from /~https://github.com/tillson/git-hound.")
 		}
-		if !app.GetFlags().ResultsOnly && !app.GetFlags().JsonOutput {
-			color.Green("Finished.")
+
+		app.GetFlags().TextRegexes = allRules
+
+		// fmt.Println(app.GetFlags().TextRegexes)
+
+		if app.GetFlags().SearchType == "ui" && viper.GetString("github_username") == "" {
+			color.Red("[!] GitHound run in UI mode but github_username not specified in config.yml. Update config.yml or run in API mode (flag: `--search-type api`)")
+			os.Exit(1)
+		} else if app.GetFlags().SearchType == "api" && viper.GetString("github_access_token") == "" {
+			color.Red("[!] GitHound run in API mode but github_access_token not specified in config.yml. Update config.yml or run in UI mode (flag: `--search-type ui`)")
+			os.Exit(1)
+		}
+
+		if app.GetFlags().SearchType == "ui" {
+			app.SearchWithUI(queries)
+		} else {
+			// fmt.Println(1)
+			app.SearchWithAPI(queries)
 		}
 
-		app.SearchWaitGroup.Wait()
 	},
 }
 
+func LoadRegexFile(path string) []app.Rule {
+	file, err := os.OpenFile(path, os.O_RDONLY, 0600)
+	if err != nil {
+		color.Yellow("[!} Error opening rules file %v: %v", app.GetFlags().RegexFile+"", err)
+	}
+	defer file.Close()
+
+	dec := yaml.NewDecoder(file)
+	ruleConfig := app.RuleConfig{}
+	err = dec.Decode(&ruleConfig)
+	if err != nil {
+		_, err := toml.DecodeFile(path, &ruleConfig)
+
+		if err != nil {
+			// fmt.Println("Resorting to .txt")
+			file, _ := os.Open(path)
+			defer file.Close()
+			scanner := bufio.NewScanner(file)
+			idCount := 1
+
+			for scanner.Scan() {
+				line := scanner.Text()
+				// fmt.Println(line)
+				// Assuming each line is a regex pattern, we create a Rule from it
+				compiled, err := pcre.Compile(line, 0)
+				if err != nil {
+					fmt.Printf("Unable to parse regex `%s` in TXT file.\n", line)
+					continue
+				}
+
+				// Create a new rule
+				rule := app.Rule{
+					ID:             fmt.Sprintf("Rule-%d", idCount), // Incremental ID
+					Pattern:        app.RegexWrapper{RegExp: compiled},
+					StringPattern:  line,                                            // Store the original pattern as StringPattern
+					Description:    fmt.Sprintf("Description for Rule-%d", idCount), // Incremental description
+					SmartFiltering: false,                                           // Default to false, you can modify if needed
+				}
+
+				// Add the rule to the config
+				ruleConfig.Rules = append(ruleConfig.Rules, rule)
+
+				idCount++ // Increment the rule ID counter
+			}
+		} else {
+			// Convert StringPattern to Pattern for TOML
+			for i, rule := range ruleConfig.Rules {
+				if rule.StringPattern != "" {
+					compiled, err := pcre.Compile(rule.StringPattern, 0)
+					if err != nil {
+						// fmt.Println("Unable to parse regex `" + rule.StringPattern + "` in TOML file.")
+					}
+					ruleConfig.Rules[i].Pattern = app.RegexWrapper{RegExp: compiled}
+
+				}
+			}
+			// fmt.Println("Parsed as TOML")
+		}
+	} else {
+		// fmt.Println("Parsed as YML")
+	}
+	// fmt.Println(2)
+	return ruleConfig.Rules
+
+}
+
 func getScanner(args []string) *bufio.Scanner {
 	if len(args) == 2 {
 		if args[0] == "searchKeyword" {
@@ -147,10 +234,10 @@ func ReadConfig() {
 	err := viper.ReadInConfig()
 	if err != nil {
 		if app.GetFlags().ConfigFile != "" {
-			color.Red("[!] '" + app.GetFlags().ConfigFile + "' was not found.")
+			color.Red("[!] Config file '" + app.GetFlags().ConfigFile + "' was not found. Please specify a correct config path with `--config-file`.")
 
 		} else {
-			color.Red("[!] config.yml was not found.")
+			color.Red("[!] config.yml was not found. Please ensure config.yml exists in current working directory or $HOME/.githound/, or use flag `--config [config_path]`.")
 		}
 		os.Exit(1)
 		return

config.example.yml

@@ -2,8 +2,14 @@
 # DO NOT CHECK YOUR USERNAME AND PASSWORD INTO GIT!
 
 # Required
-github_username: "username"
-github_password: "your_password"
+# Use with `--search-type api` flag
+github_access_token: "API_TOKEN"
+
+# Use with `--search-type ui` flag
+#github_username: "username"
+#github_password: "your_password"
+
+
 
 # Optional (comment out if not using)
 # github_totp_seed: "ABCDEF1234567890" # Obtained via /~https://github.com/settings/security