Merge pull request #4 from PaperMtn/release/1.4.0

Release/1.4.0
PaperMtn · Dec 24, 2020 · 959f034 · 959f034
2 parents 5755c8f + 209bfe0
commit 959f034
Show file tree

Hide file tree

Showing 34 changed files with 633 additions and 392 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,29 +1,49 @@
+## 1.4.0 - 2020-12-24
+### Added:
+- Refactor of rules into directories for easier management
+- Multiprocessing implemented for searching for matches. GitLab Watchman now splits regex filtering between the cores available on the device, meaning the more cores you have, the faster searching should run.
+- Handling for GitLab API rate limiting, backing off when the rate limit is hit. The rate limit may be more likely to come into effect with multiprocessing
+- Rules added to search for:
+  - Cloudflare tokens
+  - Facebook API tokens
+  - GitHub API tokens
+  - Mailchimp API tokens
+  - Mailgun API tokens
+  - Shodan API tokens
+  - Stripe API tokens
+  - Twilio API tokens
+  - Microsoft NuGet keys
+
+
 ## 1.3.0 - 2020-12-12
-###Added:
+### Added:
 - Add more information about the namespaces a project is in to logs
 - Added details owner of that namespace, for groups and users
 - Time based searching now looks at the time a file was committed, not when a project was active, which greatly reduces multiples of the same detection because a project is active but a file has not been modified.
 - Rules added:
-    - SSH private keys 
+    - SSH private keys
     - Mastercard datacash tokens
     - Heroku tokens
     - PagerDuty tokens
-  
-###Removed:
-- Enhanced logging that includes nested information, such as namespace owners, means that CSV logging is no longer practical. CSV logging has been removed and JSON via STDOUT is now the default option. 
+
+### Removed:
+- Enhanced logging that includes nested information, such as namespace owners, means that CSV logging is no longer practical. CSV logging has been removed and JSON via STDOUT is now the default option.
 
 ## 1.2.0 - 2020-11-16
 ### Added:
 - More data on namespaces added to logs
 - Better search queries for existing rules to filter out false positives
+
 ### Removed:
 - CICD variable search no longer works due to GitLab API now only allowing owners of a project to search it for variables. It has been removed.
+
 ### Fixed:
 - Bug on outputting match string for blobs/wiki-blobs
 
 ## 1.1.0 - 2020-11-14
 ### Fixed
 - Retry added for occasional Requests HTTPSConnectionPool error
+
 ### Added
 - Exact regex string match added to output from message searches
 

diff --git a/gitlab_watchman/__init__.py b/gitlab_watchman/__init__.py
@@ -13,7 +13,6 @@
 import gitlab_watchman.config as cfg
 import gitlab_watchman.logger as logger
 
-
 RULES_PATH = (Path(__file__).parent / 'rules').resolve()
 OUTPUT_LOGGER = ''
 
@@ -34,70 +33,16 @@ def search(gitlab_connection, rule, tf, scope):
     else:
         print = builtins.print
     try:
-        if scope == 'blobs':
-            print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'),
-                                                          'blobs'), 'yellow'))
-
-            blobs = gitlab.search_blobs(gitlab_connection, OUTPUT_LOGGER, rule, tf)
-            if blobs:
-                for log_data in blobs:
-                    OUTPUT_LOGGER.log_notification(log_data, 'blobs', rule.get('meta').get('name'),
-                                                   rule.get('meta').get('severity'))
-                print('Results output to log')
-
-        if scope == 'commits':
-            print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'),
-                                                          'commits'), 'yellow'))
+        print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'), scope), 'yellow'))
 
-            commits = gitlab.search_commits(gitlab_connection, OUTPUT_LOGGER, rule, tf)
-            if commits:
-                for log_data in commits:
-                    OUTPUT_LOGGER.log_notification(log_data, 'commits', rule.get('meta').get('name'),
-                                                   rule.get('meta').get('severity'))
-                print('Results output to log')
-
-        if scope == 'issues':
-            print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'),
-                                                          'issues'), 'yellow'))
-
-            issues = gitlab.search_issues(gitlab_connection, OUTPUT_LOGGER, rule, tf)
-            if issues:
-                for log_data in issues:
-                    OUTPUT_LOGGER.log_notification(log_data, 'issues', rule.get('meta').get('name'),
-                                                   rule.get('meta').get('severity'))
-                print('Results output to log')
-
-        if scope == 'wiki_blobs':
-            print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'),
-                                                          'wiki blobs'), 'yellow'))
-
-            wiki_blobs = gitlab.search_wiki_blobs(gitlab_connection, OUTPUT_LOGGER, rule, tf)
-            if wiki_blobs:
-                for log_data in wiki_blobs:
-                    OUTPUT_LOGGER.log_notification(log_data, 'wiki_blobs', rule.get('meta').get('name'),
-                                                   rule.get('meta').get('severity'))
-                print('Results output to log')
-
-        if scope == 'merge_requests':
-            print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'),
-                                                          'merge requests'), 'yellow'))
-
-            merge_requests = gitlab.search_merge_requests(gitlab_connection, OUTPUT_LOGGER, rule, tf)
-            if merge_requests:
-                for log_data in merge_requests:
-                    OUTPUT_LOGGER.log_notification(log_data, 'merge_requests', rule.get('meta').get('name'),
-                                                   rule.get('meta').get('severity'))
-                print('Results output to log')
-
-        if scope == 'milestones':
-            print(colored('Searching for {} in {}'.format(rule.get('meta').get('name'),
-                                                          'milestones'), 'yellow'))
-            milestones = gitlab.search_milestones(gitlab_connection, OUTPUT_LOGGER, rule, tf)
-            if milestones:
-                for log_data in milestones:
-                    OUTPUT_LOGGER.log_notification(log_data, 'milestones', rule.get('meta').get('name'),
+        results = gitlab.search(gitlab_connection, OUTPUT_LOGGER, rule, scope, tf)
+        if results:
+            if isinstance(OUTPUT_LOGGER, logger.CSVLogger):
+                OUTPUT_LOGGER.write_csv('exposed_{}'.format(rule.get('filename').split('.')[0]), scope, results)
+            else:
+                for log_data in results:
+                    OUTPUT_LOGGER.log_notification(log_data, scope, rule.get('meta').get('name'),
                                                    rule.get('meta').get('severity'))
-                print('Results output to log')
     except Exception as e:
         if isinstance(OUTPUT_LOGGER, logger.StdoutLogger):
             print = OUTPUT_LOGGER.log_critical
@@ -108,14 +53,18 @@ def search(gitlab_connection, rule, tf, scope):
 
 
 def load_rules():
+    """Import YAML rules"""
+
     rules = []
     try:
-        for file in os.scandir(RULES_PATH):
-            if file.name.endswith('.yaml'):
-                with open(file) as yaml_file:
-                    rule = yaml.safe_load(yaml_file)
-                    if rule.get('enabled'):
-                        rules.append(rule)
+        for root, dirs, files in os.walk(RULES_PATH):
+            for rule in files:
+                rule_path = (Path(root) / rule).resolve()
+                if rule_path.name.endswith('.yaml'):
+                    with open(rule_path) as yaml_file:
+                        rule = yaml.safe_load(yaml_file)
+                        if rule.get('enabled'):
+                            rules.append(rule)
         return rules
     except Exception as e:
         if isinstance(OUTPUT_LOGGER, logger.StdoutLogger):