Merge branch 'release-1.1.8'

Conflicts:
	mtools/mlogvis/mlogvis.py

CHANGES.md

@@ -1,6 +1,20 @@
 Changes to mtools
 =================
 
+#### version 1.1.8
+
+  * mloginfo: storage engine is now listed for log files (#330)
+  * mplotqueries: x-axis bounds corrected when parsing multiple files (#322)
+  * mlogfilter: truncated log lines ("too long ...") recognized and parsed as much as possible (#133)
+  * better cross-platform script support, especially for window susers (#230)
+  * logging components are updated to match final version of MongoDB 3.0 (#328, #327)
+  * removed hard dependency on pymongo, only required if mlaunch is used (#337)
+  * removed deprecated scripts like mlogversion, mlogdistinct (#336)
+  * command in LogEvent is now always lowercase (#335)
+  * LogEvent now has writeConflicts property (#334)
+  * documented matplotlib minimum version 1.8.0 (#332)
+
+
 #### version 1.1.7
 
   * mtools now understands 2.8 style log format, with severity and components. Added by @jimoleary (#269)

INSTALL.md

@@ -104,8 +104,7 @@ Installation instructions for matplotlib can be found under the [matplotlib Inst
 
 *required for matplotlib (in mplotqueries)*
 
-[NumPy](http://numpy.scipy.org/) is a Python module for scientific computing and numerical calculations.
-Try installing NumPy with pip, by doing:
+[NumPy](http://numpy.scipy.org/) is a Python module for scientific computing and numerical calculations. Version 1.8.0 or higher is required for mtools. Try installing NumPy with pip, by doing:
 
     pip install numpy
 

MANIFEST.in

@@ -1,5 +1,4 @@
 include *.md
 recursive-include tutorials *
-recursive-include scripts *
 recursive-include mtools *.py
 recursive-include mtools/data *

README.md

@@ -50,7 +50,7 @@ instructions for these modules.
 Recent Changes
 --------------
 
-The current version of mtools is 1.1.7. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools.
+The current version of mtools is 1.1.8. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools.
 
 
 Contribute to mtools

mtools/mgenerate/mgenerate.py

@@ -217,7 +217,9 @@ def run(self, arguments=None):
             p.join()
 
 
-
-if __name__ == '__main__':
+def main():
     tool = MGeneratorTool()
     tool.run()
+
+if __name__ == '__main__':
+    sys.exit(main())

mtools/mgenerate/mgenerate_spec.md

@@ -121,7 +121,7 @@ Will not insert the key/value pair. A percentage of missing values can be specif
 
 
 ###### Missing Percentage
-`{ "$missing" : { "percentage" : 30, "ifnot" : VALUE } }` <br>
+`{ "$missing" : { "percent" : 30, "ifnot" : VALUE } }` <br>
 
 Will cause the key/value pair to be missing 30% of the time, and otherwise set the VALUE for the given key.
 

mtools/mgenerate/operators/operators.py

@@ -203,7 +203,7 @@ class CoordinateOperator(BaseOperator):
 
     dict_format = True
     string_format = True
-    names = ['$coordinates', '$coordinate', '$coord']
+    names = ['$coordinates', '$coordinate', '$coord', '$geo']
     defaults = OrderedDict([ ('long_lim', [-180, 180]), ('lat_lim', [-90, 90]) ])
 
     def __call__(self, options=None):

mtools/mlaunch/mlaunch.py

@@ -92,6 +92,10 @@ def shutdown_host(port, username=None, password=None, authdb=None):
             mc.admin.command('shutdown', force=True)
         except AutoReconnect:
             pass
+        except OperationFailure: 
+            print "Error: cannot authenticate to shut down %s." % host
+            return
+
     except ConnectionFailure:
         pass
     else:
@@ -162,7 +166,7 @@ def run(self, arguments=None):
         init_parser.add_argument('--name', action='store', metavar='NAME', default='replset', help='name for replica set (default=replset)')
 
         # sharded clusters
-        init_parser.add_argument('--sharded', action='store', nargs='+', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards.')
+        init_parser.add_argument('--sharded', '--shards', action='store', nargs='+', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards.')
         init_parser.add_argument('--config', action='store', default=1, type=int, metavar='NUM', choices=[1, 3], help='adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)')
         init_parser.add_argument('--mongos', action='store', default=1, type=int, metavar='NUM', help='starts NUM mongos processes (requires --sharded, default=1)')
 
@@ -239,7 +243,6 @@ def run(self, arguments=None):
 
 
     # -- below are the main commands: init, start, stop, list, kill
-
     def init(self):
         """ sub-command init. Branches out to sharded, replicaset or single node methods. """
 
@@ -278,7 +281,7 @@ def init(self):
 
             # start mongod (shard and config) nodes and wait
             nodes = self.get_tagged(['mongod', 'down'])
-            self._start_on_ports(nodes, wait=True)
+            self._start_on_ports(nodes, wait=True, overrideAuth=True)
 
             # initiate replica sets if init is called for the first time
             if first_init:
@@ -289,7 +292,7 @@ def init(self):
 
             # add mongos
             mongos = sorted(self.get_tagged(['mongos', 'down']))
-            self._start_on_ports(mongos, wait=True)
+            self._start_on_ports(mongos, wait=True, overrideAuth=True)
 
             if first_init:
                 # add shards
@@ -331,8 +334,8 @@ def init(self):
                                 print res, '- will retry'
 
                     time.sleep(1)
-
 
+
         elif self.args['single']:
             # just start node
             nodes = self.get_tagged(['single', 'down'])
@@ -373,7 +376,7 @@ def init(self):
                 raise RuntimeError("can't connect to server, so adding admin user isn't possible")
 
             if "clusterAdmin" not in self.args['auth_roles']:
-                warnings.warn("the stop command will not work with auth if the user does not have the clusterAdmin role")
+                warnings.warn("the stop command will not work with auth because the user does not have the clusterAdmin role")
 
             self._add_user(sorted(nodes)[0], name=self.args['username'], password=self.args['password'], 
                 database=self.args['auth_db'], roles=self.args['auth_roles'])
@@ -391,7 +394,6 @@ def init(self):
             authdb = self.args['auth_db'] if self.args['auth'] else None
             shutdown_host(port, username, password, authdb)
 
-
         # write out parameters
         if self.args['verbose']:
             print "writing .mlaunch_startup file."
@@ -400,6 +402,12 @@ def init(self):
         # discover again, to get up-to-date info
         self.discover()
 
+        # for sharded authenticated clusters, restart after first_init to enable auth
+        if self.args['sharded'] and self.args['auth'] and first_init:
+            if self.args['verbose']:
+                print "restarting cluster to enable auth..."
+            self.restart()
+
         if self.args['verbose']:
             print "done."
 
@@ -939,6 +947,8 @@ def _create_paths(self, basedir, name=None):
 
     def _get_ports_from_args(self, args, extra_tag):
         tags = []
+        if 'tags' not in args:
+            args['tags'] = []
 
         for tag1, tag2 in zip(args['tags'][:-1], args['tags'][1:]):
             if re.match('^\d{1,2}$', tag1):
@@ -992,6 +1002,10 @@ def _filter_valid_arguments(self, arguments, binary="mongod", config=False):
                     continue
                 accepted_arguments.append(argument)
 
+        # add undocumented option
+        if binary == "mongod":
+            accepted_arguments.append('--wiredTigerEngineConfigString')
+
         # filter valid arguments
         result = []
         for i, arg in enumerate(arguments):
@@ -1029,12 +1043,19 @@ def _get_shard_names(self, args):
         return shard_names
 
 
-
-    def _start_on_ports(self, ports, wait=False):
+    def _start_on_ports(self, ports, wait=False, overrideAuth=False):
         threads = []
 
+        if overrideAuth and self.args['verbose']:
+            print "creating cluster without auth for setup, will enable auth at the end..."
+
         for port in ports:
             command_str = self.startup_info[str(port)]
+
+            if overrideAuth:
+                # this is to set up sharded clusters without auth first, then relaunch with auth
+                command_str = re.sub(r'--keyFile \S+', '', command_str)
+
             ret = subprocess.call([command_str], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True)
 
             binary = command_str.split()[0]
@@ -1097,15 +1118,15 @@ def _get_processes(self):
 
         for p in psutil.process_iter():
             # skip all but mongod / mongos
-            if p.name not in ['mongos', 'mongod']:
+            if p.name() not in ['mongos', 'mongod']:
                 continue
 
             port = None
             for possible_port in self.startup_info:
                 # compare ports based on command line argument
                 startup = self.startup_info[possible_port].split()
                 try:
-                    p_port = p.cmdline[p.cmdline.index('--port')+1]
+                    p_port = p.cmdline()[p.cmdline().index('--port')+1]
                     startup_port = startup[startup.index('--port')+1]
                 except ValueError:
                     continue
@@ -1292,9 +1313,15 @@ def _construct_mongos(self, logpath, port, configdb):
         # store parameters in startup_info
         self.startup_info[str(port)] = command_str
 
+
+    def _read_key_file():
+        with open(os.path.join(self.dir, 'keyfile'), 'r') as f:
+            return ''.join(f.readlines())
 
 
-
-if __name__ == '__main__':
+def main():
     tool = MLaunchTool()
     tool.run()
+
+if __name__ == '__main__':
+    sys.exit(main())

mtools/mlogfilter/mlogfilter.py

@@ -281,7 +281,10 @@ def run(self, arguments=None):
                     if sys.stdin.isatty():
                         break
 
-if __name__ == '__main__':
 
+def main():
     tool = MLogFilterTool()
     tool.run()
+
+if __name__ == '__main__':
+    sys.exit(main())

mtools/mloginfo/mloginfo.py

@@ -51,7 +51,6 @@ def run(self, arguments=None):
             print "     length: %s" % len(self.logfile)
             print "     binary: %s" % (self.logfile.binary or "unknown")
 
-
             version = (' -> '.join(self.logfile.versions) or "unknown")
 
             # if version is unknown, go by date
@@ -63,12 +62,12 @@ def run(self, arguments=None):
                 elif self.logfile.datetime_format == "iso8601-utc" or \
                      self.logfile.datetime_format == "iso8601-local":
                     if self.logfile.has_level:
-                        version = '>= 2.8 (iso8601 format, level, component)'
+                        version = '>= 3.0 (iso8601 format, level, component)'
                     else:
                         version = '= 2.6.x (iso8601 format)'
 
-            print "    version: %s" % version,
-            print
+            print "    version: %s" % version
+            print "    storage: %s" % (self.logfile.storage_engine or 'unknown')
 
             # now run all sections
             for section in self.sections:
@@ -78,6 +77,9 @@ def run(self, arguments=None):
                     section.run()
 
 
-if __name__ == '__main__':
+def main():
     tool = MLogInfoTool()
     tool.run()
+
+if __name__ == '__main__':
+    sys.exit(main())

mtools/mloginfo/sections/connection_section.py

@@ -2,8 +2,11 @@
 from collections import defaultdict
 import re
 
-from mtools.util.profile_collection import ProfileCollection
-
+try: 
+    from mtools.util.profile_collection import ProfileCollection
+except ImportError:
+    ProfileCollection = None
+
 class ConnectionSection(BaseSection):
     """ This section goes through the logfile and extracts information 
         about opened and closed connections.

mtools/mloginfo/sections/distinct_section.py

@@ -1,7 +1,11 @@
 from base_section import BaseSection
 
 from mtools.util.log2code import Log2CodeConverter
-from mtools.util.profile_collection import ProfileCollection
+
+try: 
+    from mtools.util.profile_collection import ProfileCollection
+except ImportError:
+    ProfileCollection = None
 
 from collections import defaultdict
 

mtools/mloginfo/sections/query_section.py

@@ -1,6 +1,5 @@
 from base_section import BaseSection
 
-from mtools.util.profile_collection import ProfileCollection
 from mtools.util.grouping import Grouping
 from mtools.util.print_table import print_table
 from mtools.util import OrderedDict

mtools/mloginfo/sections/restart_section.py

@@ -1,6 +1,9 @@
 from base_section import BaseSection
 
-from mtools.util.profile_collection import ProfileCollection
+try: 
+    from mtools.util.profile_collection import ProfileCollection
+except ImportError:
+    ProfileCollection = None
 
 class RestartSection(BaseSection):
     """ This section determines if there were any restarts in the log file and prints out

mtools/mlogvis/mlogvis.py

@@ -18,6 +18,7 @@ def __init__(self):
             a browser. Automatically opens a browser tab and shows the file.'
         self.argparser.add_argument('--no-browser', action='store_true', help='only creates .html file, but does not open the browser.')
         self.argparser.add_argument('--out', '-o', action='store', default=None, help='filename to output. Default is <original logfile>.html')
+        self.argparser.add_argument('--line-max', action='store', default=10000, help='max count of datapoints at which actual log line strings are not printed any more.')
 
 
     def _export(self, with_line_str=True):
@@ -33,8 +34,8 @@ def _export(self, with_line_str=True):
             if logevent.duration != None and logevent.datetime:
                 out_count += 1
                 # if too many lines include a line_str, the page won't load
-                if with_line_str and out_count > 10000:
-                    print "Warning: more than 10,000 data points detected. Skipping actual log line strings for faster plotting."
+                if with_line_str and out_count > self.args['line_max']:
+                    print "Warning: more than %i data points detected. Skipping actual log line strings for faster plotting." % self.args['line_max']
                     return False
                 # write log line out as json
                 if not first_row:
@@ -92,7 +93,9 @@ def run(self, arguments=None):
             print "serving visualization on file://"+dstfilelocation
             webbrowser.open("file://"+dstfilelocation)
 
-
-if __name__ == '__main__':
+def main():
     tool = MLogVisTool()
     tool.run()
+
+if __name__ == '__main__':
+    sys.exit(main())