Cleaning the code base a bit

.gitignore

@@ -153,4 +153,4 @@ __MACOSX
 
 *~test
 *.apk
-dumps/android_scan.logs
+phone_dumps/dumps/android_scan.logs

ios_permissions.json

@@ -1 +1 @@
-{"NSCalendarsUsageDescription": "Calendars", "NSVideoSubscriberAccountUsageDescription": "VideoSubscriberAccount", "kTCCServiceTencentWeibo": "TencentWeibo", "NSVoIPUsageDescription": "VoIP", "NSPhotoLibraryAddUsageDescription": "PhotoLibraryAdd", "kTCCServiceBluetoothPeripheral": "Bluetooth", "kTCCServiceMotion": "Motion", "kTCCServiceCamera": "Camera", "kTCCServiceSiri": "Siri", "NSRemindersUsageDescription": "Reminders", "NSLocationWhenInUseUsageDescription": "LocationWhenInUse", "NSLocationAlwaysUsageDescription": "LocationAlways", "kTCCServiceMediaLibrary": "MediaLibrary", "NSAppleMusicUsageDescription": "AppleMusic", "kTCCServiceWillow": "Willow", "kTCCServiceReminders": "Reminders", "kTCCServiceFaceID": "FaceID", "kTCCServiceCalendar": "Calendars", "NSBluetoothPeripheralUsageDescription": "Bluetooth", "kTCCServiceUbiquity": "iCloud (Ubiquity)", "kTCCServiceCalls": "Calls", "kTCCServiceLiverpool": "Liverpool", "NSMotionUsageDescription": "Motion", "kTCCServiceAddressBook": "AddressBook", "kTCCServiceKeyboardNetwork": "KeyboardNetwork", "NFCReaderUsageDescription": "NFCReader", "NSMicrophoneUsageDescription": "Microphone", "NSHealthUpdateUsageDescription": "HealthUpdate", "NSLocationUsageDescription": "Location", "NSHealthShareUsageDescription": "HealthShare", "kTCCServiceFacebook": "Facebook", "NSFaceIDUsageDescription": "FaceID", "NSContactsUsageDescription": "Contacts", "kTCCServiceTwitter": "Twitter", "kTCCServiceSinaWeibo": "SinaWeibo", "kTCCServiceMicrophone": "Microphone", "NSSpeechRecognitionUsageDescription": "SpeechRecognition", "NSSiriUsageDescription": "Siri", "NSHomeKitUsageDescription": "HomeKit", "kTCCServiceAll": "All", "kTCCServicePhotosAdd": "PhotosAdd", "kTCCServiceMSO": "MSO", "NSCameraUsageDescription": "Camera", "NSPhotoLibraryUsageDescription": "PhotoLibrary", "kTCCServiceSpeechRecognition": "SpeechRecognition", "kTCCServicePhotos": "Photos", "kTCCServiceShareKit": "ShareKit", "'": "'", "c": "c", "o": "o", "m": "m", ".": ".", "a": "a", "p": "p", "l": "l", "e": "e", "r": "r", "i": "i", "v": "v", "t": "t", "w": "w", "d": "d", "b": "b"}
+{"NSCalendarsUsageDescription": "Calendars", "NSVideoSubscriberAccountUsageDescription": "VideoSubscriberAccount", "kTCCServiceTencentWeibo": "TencentWeibo", "NSVoIPUsageDescription": "VoIP", "NSPhotoLibraryAddUsageDescription": "PhotoLibraryAdd", "kTCCServiceBluetoothPeripheral": "Bluetooth", "kTCCServiceMotion": "Motion", "kTCCServiceCamera": "Camera", "kTCCServiceSiri": "Siri", "NSRemindersUsageDescription": "Reminders", "NSLocationWhenInUseUsageDescription": "LocationWhenInUse", "NSLocationAlwaysUsageDescription": "LocationAlways", "kTCCServiceMediaLibrary": "MediaLibrary", "NSAppleMusicUsageDescription": "AppleMusic", "kTCCServiceWillow": "Willow", "kTCCServiceReminders": "Reminders", "kTCCServiceFaceID": "FaceID", "kTCCServiceCalendar": "Calendars", "NSBluetoothPeripheralUsageDescription": "Bluetooth", "kTCCServiceUbiquity": "iCloud (Ubiquity)", "kTCCServiceCalls": "Calls", "kTCCServiceLiverpool": "Liverpool", "NSMotionUsageDescription": "Motion", "kTCCServiceAddressBook": "AddressBook", "kTCCServiceKeyboardNetwork": "KeyboardNetwork", "NFCReaderUsageDescription": "NFCReader", "NSMicrophoneUsageDescription": "Microphone", "NSHealthUpdateUsageDescription": "HealthUpdate", "NSLocationUsageDescription": "Location", "NSHealthShareUsageDescription": "HealthShare", "kTCCServiceFacebook": "Facebook", "NSFaceIDUsageDescription": "FaceID", "NSContactsUsageDescription": "Contacts", "kTCCServiceTwitter": "Twitter", "kTCCServiceSinaWeibo": "SinaWeibo", "kTCCServiceMicrophone": "Microphone", "NSSpeechRecognitionUsageDescription": "SpeechRecognition", "NSSiriUsageDescription": "Siri", "NSHomeKitUsageDescription": "HomeKit", "kTCCServiceAll": "All", "kTCCServicePhotosAdd": "PhotosAdd", "kTCCServiceMSO": "MSO", "NSCameraUsageDescription": "Camera", "NSPhotoLibraryUsageDescription": "PhotoLibrary", "kTCCServiceSpeechRecognition": "SpeechRecognition", "kTCCServicePhotos": "Photos", "kTCCServiceShareKit": "ShareKit", "kTCCServiceListenEvent": "ListenEvent", "kTCCServiceVoiceBanking": "VoiceBanking", "'": "'", "c": "c", "o": "o", "m": "m", ".": ".", "a": "a", "p": "p", "l": "l", "e": "e", "r": "r", "i": "i", "v": "v", "t": "t", "w": "w", "d": "d", "b": "b"}

isdi

@@ -6,7 +6,7 @@ import webbrowser
 from threading import Timer
 
 import config
-from db import init_db
+from phone_scanner import db
 from web import app, sa
 
 PORT = 6200 if not (config.TEST or config.DEBUG) else 6202
@@ -28,7 +28,7 @@ if __name__ == "__main__":
               "SQL_DB: {config.SQL_DB_PATH}")
 
     print(f"TEST={config.TEST}")
-    init_db(app, sa, force=config.TEST)
+    db.init_db(app, sa, force=config.TEST)
     config.setup_logger()
     Timer(1, open_browser).start()
     app.run(host=HOST, port=PORT, debug=config.DEBUG, use_reloader=config.DEBUG)

phone_scanner.py → phone_scanner/__init__.py

@@ -6,14 +6,14 @@
 import shlex
 import sqlite3
 import sys
-import blocklist
 import config
-import parse_dump
 import pandas as pd
 from collections import defaultdict
 from datetime import datetime
-from android_permissions import all_permissions
-from runcmd import catch_err, run_command
+from . import blocklist
+from . import parse_dump
+from .android_permissions import all_permissions
+from .runcmd import catch_err, run_command
 
 
 class AppScan(object):

android_permissions.py → phone_scanner/android_permissions.py

@@ -3,12 +3,11 @@
 """
 
 from rsonlite import simpleparse
-from runcmd import run_command, catch_err
 import pandas as pd
 import datetime
 import config
 import re
-import json
+from .runcmd import run_command, catch_err
 
 #MAP = config.ANDROID_PERMISSIONS
 DUMPPKG = 'dumppkg'
@@ -20,7 +19,7 @@ def _parse_time(time_str):
     Modified from virhilo's answer at https://stackoverflow.com/a/4628148/851699
     :param time_str: A string identifying a duration.  (eg. 2h13m)
     :return datetime.timedelta: A datetime.timedelta object
-        r'^+((?P<days>[\.\d]+?)d)?((?P<hours>[\.\d]+?)h)?((?P<minutes>[\.\d]+?)m)?((?P<seconds>[\.\d]+?)s)?((?P<milliseconds>[\.\d]+?)ms)?')
+        r'^+((?P<days>[\\.\d]+?)d)?((?P<hours>[\.\d]+?)h)?((?P<minutes>[\.\d]+?)m)?((?P<seconds>[\.\d]+?)s)?((?P<milliseconds>[\\.\d]+?)ms)?')
     """
     timedelta_re = re.compile(
         r'^.((?P<days>[\.\d]+?)d)?((?P<hours>[\.\d]+?)h)?((?P<minutes>[\.\d]+?)m)?((?P<seconds>[\.\d]+?)s)?((?P<milliseconds>[\.\d]+?)ms)?')

parse_dump.py → phone_scanner/parse_dump.py

@@ -116,7 +116,7 @@ def prune_empty_keys(d):
     return d
 
 
-def retrieve(dict_, nest):
+def retrieve(dict_: pd.DataFrame, nest: list) -> str:
     """
     Navigates dictionaries like dict_[nest0][nest1][nest2]...
     gracefully.
@@ -125,9 +125,11 @@ def retrieve(dict_, nest):
     try:
         return reduce(operator.getitem, nest, dict_)
     except KeyError as e:
-        return f"{e}"
+        print(f"KeyError: {e} for dict_={dict_} and nest={nest}")
+        return ""
     except TypeError as e:
-        return f"{e}"
+        print(f"TypeError: {e} for dict_={dict_} and nest={nest}")
+        return ""
 
 
 class PhoneDump(object):
@@ -490,12 +492,9 @@ def load_file(self):
 
     def check_unseen_permissions(self, permissions):
         for permission in permissions:
+            if not permission: continue  # Empty permission, skip
             if permission not in self.permissions_map:
-                print(
-                    "Have not seen "
-                    + str(permission)
-                    + " before. Making note of this..."
-                )
+                print(f"Have not seen {permission} before. Making note of this...")
                 permission_human_readable = permission.replace("kTCCService", "")
                 with open(
                     os.path.join(config.THIS_DIR, "ios_permissions.json"), "w"

web/view/consult.py

@@ -4,7 +4,7 @@
 from web.forms import ClientForm
 from flask import render_template, request, session, redirect, url_for
 from phone_scanner import AndroidScan, IosScan, TestScan
-from db import get_client_devices_from_db, new_client_id
+from phone_scanner.db import get_client_devices_from_db, new_client_id
 
 @app.route('/form/', methods=['GET', 'POST'])
 def client_forms():

web/view/index.py

@@ -2,7 +2,7 @@
 from web import app
 from flask import render_template, request, session
 from phone_scanner import AndroidScan, IosScan, TestScan
-from db import get_client_devices_from_db, new_client_id
+from phone_scanner.db import get_client_devices_from_db, new_client_id
 
 # FIXME: why are we scanning devices before people clicked on scan now?
 android = AndroidScan()

web/view/privacy.py

@@ -1,7 +1,7 @@
 from flask import request, render_template
 from web import app
 from web.view import get_device
-from privacy_scan_android import do_privacy_check
+from phone_scanner.privacy_scan_android import do_privacy_check
 import config
 
 @app.route("/privacy", methods=['GET'])

web/view/results.py

@@ -1,7 +1,7 @@
 from web import app
 from web.view.index import get_device
 from flask import request, render_template, redirect, url_for
-from db import get_scan_res_from_db, get_app_info_from_db, first_element_or_none
+from phone_scanner.db import get_scan_res_from_db, get_app_info_from_db, first_element_or_none
 
 @app.route("/view_results", methods=['POST', 'GET'])
 def view_results():

web/view/save.py

@@ -1,7 +1,7 @@
 from flask import request, session
 import config
 from web import app
-from db import get_serial_from_db, save_note, update_appinfo, update_mul_appinfo, create_report, get_device_from_db
+from phone_scanner.db import get_serial_from_db, save_note, update_appinfo, update_mul_appinfo, create_report, get_device_from_db
 from web.view.index import get_device
 
 

web/view/scan.py

@@ -4,8 +4,8 @@
 from web import app
 from web.view.index import get_device
 from flask import render_template, request, session, redirect, url_for
-import db
-from db import (get_client_devices_from_db, new_client_id, create_scan, 
+from phone_scanner import db
+from phone_scanner.db import (get_client_devices_from_db, new_client_id, create_scan, 
                 create_mult_appinfo, first_element_or_none)