Merge pull request #427 from Steve-Hand/changes_408_404_426

Fixed 408 and 426. Enabled mtools utilities to be run under debugger …

mtools/mloginfo/mloginfo.py

@@ -81,6 +81,7 @@ def run(self, arguments=None):
 def main():
     tool = MLogInfoTool()
     tool.run()
+    return 0 # we need to return an integer
 
 if __name__ == '__main__':
     sys.exit(main())

mtools/test/logfiles/mongod_328.log

@@ -0,0 +1,29 @@
+2016-08-14T14:36:30.641-0500 I CONTROL  [initandlisten] MongoDB starting : pid=78886 port=28000 dbpath=/Users/shand/Documents/Development/mtools_steve/mtools/data_3_2/db 64-bit host=Stevens-MBP-2.attlocal.net
+2016-08-14T14:36:30.641-0500 I CONTROL  [initandlisten] db version v3.2.8
+2016-08-14T14:36:30.641-0500 I CONTROL  [initandlisten] git version: ed70e33130c977bda0024c125b56d159573dbaf0
+2016-08-14T14:36:30.641-0500 I CONTROL  [initandlisten] allocator: system
+2016-08-14T14:36:30.642-0500 I CONTROL  [initandlisten] modules: none
+2016-08-14T14:36:30.642-0500 I CONTROL  [initandlisten] build environment:
+2016-08-14T14:36:30.642-0500 I CONTROL  [initandlisten]     distarch: x86_64
+2016-08-14T14:36:30.642-0500 I CONTROL  [initandlisten]     target_arch: x86_64
+2016-08-14T14:36:30.642-0500 I CONTROL  [initandlisten] options: { net: { port: 28000 }, processManagement: { fork: true }, storage: { dbPath: "data_3_2/db", engine: "wiredTiger" }, systemLog: { destination: "file", logAppend: true, path: "data_3_2/mongod.log" } }
+2016-08-14T14:36:30.644-0500 I STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=9G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
+2016-08-14T14:36:31.036-0500 I CONTROL  [initandlisten] 
+2016-08-14T14:36:31.036-0500 I CONTROL  [initandlisten] ** WARNING: soft rlimits too low. Number of files is 256, should be at least 1000
+2016-08-14T14:36:31.037-0500 I NETWORK  [HostnameCanonicalizationWorker] Starting hostname canonicalization worker
+2016-08-14T14:36:31.037-0500 I FTDC     [initandlisten] Initializing full-time diagnostic data capture with directory '/Users/shand/Documents/Development/mtools_steve/mtools/data_3_2/db/diagnostic.data'
+2016-08-14T14:36:31.095-0500 I NETWORK  [initandlisten] waiting for connections on port 28000
+2016-08-14T14:36:39.474-0500 I NETWORK  [initandlisten] connection accepted from 127.0.0.1:53732 #1 (1 connection now open)
+2016-08-14T14:36:45.625-0500 I NETWORK  [conn1] end connection 127.0.0.1:53732 (0 connections now open)
+2016-08-14T14:37:14.753-0500 I CONTROL  [signalProcessingThread] got signal 15 (Terminated: 15), will terminate after current cmd ends
+2016-08-14T14:37:14.753-0500 I FTDC     [signalProcessingThread] Shutting down full-time diagnostic data capture
+2016-08-14T14:37:14.756-0500 I CONTROL  [signalProcessingThread] now exiting
+2016-08-14T14:37:14.756-0500 I NETWORK  [signalProcessingThread] shutdown: going to close listening sockets...
+2016-08-14T14:37:14.756-0500 I NETWORK  [signalProcessingThread] closing listening socket: 6
+2016-08-14T14:37:14.756-0500 I NETWORK  [signalProcessingThread] closing listening socket: 7
+2016-08-14T14:37:14.757-0500 I NETWORK  [signalProcessingThread] removing socket file: /tmp/mongodb-28000.sock
+2016-08-14T14:37:14.758-0500 I NETWORK  [signalProcessingThread] shutdown: going to flush diaglog...
+2016-08-14T14:37:14.758-0500 I NETWORK  [signalProcessingThread] shutdown: going to close sockets...
+2016-08-14T14:37:14.758-0500 I STORAGE  [signalProcessingThread] WiredTigerKVEngine shutting down
+2016-08-14T14:37:14.896-0500 I STORAGE  [signalProcessingThread] shutdown: removing fs lock...
+2016-08-14T14:37:14.896-0500 I CONTROL  [signalProcessingThread] dbexit:  rc: 0

mtools/test/test_util_logfile.py

@@ -69,11 +69,17 @@ def test_storage_engine_detection(self):
         logfile = LogFile(mmapv1)
         assert logfile.storage_engine == 'mmapv1'
 
+        # test for 3.0 WT detection
         logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'wiredtiger.log')
         wiredtiger = open(logfile_path, 'r')
         logfile = LogFile(wiredtiger)
         assert logfile.storage_engine == 'wiredTiger'
 
+        # test for 3.2 WT detection
+        logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_328.log')
+        wiredtiger = open(logfile_path, 'r')
+        logfile = LogFile(wiredtiger)
+        assert logfile.storage_engine == 'wiredTiger'
 
     def test_hostname_port(self):
         # mongod

mtools/util/cmdlinetool.py

@@ -75,7 +75,15 @@ def __init__(self):
         self.argparser = argparse.ArgumentParser()
         self.argparser.add_argument('--version', action='version', version="mtools version %s" % __version__)
         self.argparser.add_argument('--no-progressbar', action='store_true', default=False, help='disables progress bar')
-        self.is_stdin = not sys.stdin.isatty()
+        """ PyCharm and other debuggers apparently run script in a non-TTY process.
+            For now, just ignore the fact that we are debugging
+        """
+        if __debug__:
+            self.is_stdin = False
+        else:
+            self.is_stdin = not sys.stdin.isatty()
+
+
 
 
     def run(self, arguments=None, get_unknowns=False):

mtools/util/logfile.py

@@ -259,6 +259,10 @@ def _iterate_lines(self):
                     self._hostname = match.group('host')
                     self._port = match.group('port')
 
+            ''' For 3.0 the "[initandlisten] options:" long entry contained the "engine" field
+                if WiredTiger was the storage engine. There were only two engines, MMAPv1
+                and WiredTiger
+            '''
             if "[initandlisten] options:" in line:
                 match = re.search('replSet: "(?P<replSet>\S+)"', line)
                 if match:
@@ -270,6 +274,13 @@ def _iterate_lines(self):
                 else:
                     self._storage_engine = 'mmapv1'
 
+            ''' For 3.2 the "[initandlisten] options:" no longer contains the "engine" field
+                So now we have to look for the "[initandlisten] wiredtiger_open config:"
+                which was present in 3.0, but would now tell us definatively that wiredTiger is being used
+            '''
+            if "[initandlisten] wiredtiger_open config:" in line:
+                self._storage_engine =  'wiredTiger'
+
             if "command admin.$cmd command: { replSetInitiate:" in line:
                 match = re.search('{ _id: "(?P<replSet>\S+)", members: (?P<replSetMembers>[^]]+ ])', line)
                 if match:
@@ -290,7 +301,7 @@ def _iterate_lines(self):
                 if tokens[1].endswith(']'):
                     pos = 4
                 else:
-                    pos = 7
+                    pos = 5
                 host = tokens[pos]
                 rs_state = tokens[-1]
                 state = (host, rs_state, LogEvent(line))