From 55fc4a636302255d87b30e86fac1cb0c8dda643c Mon Sep 17 00:00:00 2001 From: Timothy Van Heest Date: Sat, 31 May 2014 19:30:27 -0400 Subject: [PATCH 01/22] Fixed installation process to use entry_points.console_scripts to make scripts accessible in a cross-platform way --- MANIFEST.in | 1 - mtools/mgenerate/mgenerate.py | 6 ++++-- mtools/mlaunch/mlaunch.py | 7 ++++--- mtools/mlog2json/mlog2json.py | 6 +++++- mtools/mlogdistinct/mlogdistinct.py | 6 +++++- mtools/mlogfilter/mlogfilter.py | 6 ++++-- mtools/mloginfo/mloginfo.py | 5 ++++- mtools/mlogmerge/mlogmerge.py | 6 +++++- mtools/mlogversion/mlogversion.py | 6 +++++- mtools/mlogvis/mlogvis.py | 6 ++++-- mtools/mplotqueries/mplotqueries.py | 5 +++-- mtools/util/log2code.py | 0 scripts/mgenerate | 1 - scripts/mlaunch | 1 - scripts/mlog2json | 1 - scripts/mlogdistinct | 1 - scripts/mlogfilter | 1 - scripts/mloginfo | 1 - scripts/mlogmerge | 1 - scripts/mlogversion | 1 - scripts/mlogvis | 1 - scripts/mplotqueries | 1 - setup.py | 17 ++++++++++++++--- 23 files changed, 57 insertions(+), 30 deletions(-) mode change 100755 => 100644 mtools/mlaunch/mlaunch.py mode change 100755 => 100644 mtools/mlog2json/mlog2json.py mode change 100755 => 100644 mtools/mlogfilter/mlogfilter.py mode change 100755 => 100644 mtools/mlogmerge/mlogmerge.py mode change 100755 => 100644 mtools/mlogversion/mlogversion.py mode change 100755 => 100644 mtools/mlogvis/mlogvis.py mode change 100755 => 100644 mtools/mplotqueries/mplotqueries.py mode change 100755 => 100644 mtools/util/log2code.py delete mode 120000 scripts/mgenerate delete mode 120000 scripts/mlaunch delete mode 120000 scripts/mlog2json delete mode 120000 scripts/mlogdistinct delete mode 120000 scripts/mlogfilter delete mode 120000 scripts/mloginfo delete mode 120000 scripts/mlogmerge delete mode 120000 scripts/mlogversion delete mode 120000 scripts/mlogvis delete mode 120000 scripts/mplotqueries diff --git a/MANIFEST.in b/MANIFEST.in index 391dcbf4..8b13cc29 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include *.md recursive-include tutorials * -recursive-include scripts * recursive-include mtools *.py recursive-include mtools/data * diff --git a/mtools/mgenerate/mgenerate.py b/mtools/mgenerate/mgenerate.py index 457cf9d1..06c00b92 100644 --- a/mtools/mgenerate/mgenerate.py +++ b/mtools/mgenerate/mgenerate.py @@ -191,7 +191,9 @@ def run(self, arguments=None): p.join() - -if __name__ == '__main__': +def main(): tool = MGeneratorTool() tool.run() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py old mode 100755 new mode 100644 index 5c94062b..8876054f --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -1227,8 +1227,9 @@ def _construct_mongos(self, logpath, port, configdb): self.startup_info[str(port)] = command_str - - -if __name__ == '__main__': +def main(): tool = MLaunchTool() tool.run() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/mtools/mlog2json/mlog2json.py b/mtools/mlog2json/mlog2json.py old mode 100755 new mode 100644 index a6c78b59..e6bc39b1 --- a/mtools/mlog2json/mlog2json.py +++ b/mtools/mlog2json/mlog2json.py @@ -1,3 +1,7 @@ #!/usr/bin/env python -print "deprecated since version 1.1.0 of mtools. Use 'mlogfilter --json' instead." +def main(): + print "deprecated since version 1.1.0 of mtools. Use 'mlogfilter --json' instead." + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/mtools/mlogdistinct/mlogdistinct.py b/mtools/mlogdistinct/mlogdistinct.py index 9a59e67c..da54d04e 100644 --- a/mtools/mlogdistinct/mlogdistinct.py +++ b/mtools/mlogdistinct/mlogdistinct.py @@ -1,3 +1,7 @@ #!/usr/bin/env python -print "deprecated since version 1.1.0 of mtools. Use 'mloginfo --distinct' instead." +def main(): + print "deprecated since version 1.1.0 of mtools. Use 'mloginfo --distinct' instead." + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/mtools/mlogfilter/mlogfilter.py b/mtools/mlogfilter/mlogfilter.py old mode 100755 new mode 100644 index 59c399e7..41fbfdee --- a/mtools/mlogfilter/mlogfilter.py +++ b/mtools/mlogfilter/mlogfilter.py @@ -282,7 +282,9 @@ def run(self, arguments=None): break -if __name__ == '__main__': - +def main(): tool = MLogFilterTool() tool.run() + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/mtools/mloginfo/mloginfo.py b/mtools/mloginfo/mloginfo.py index 921ae052..572519af 100644 --- a/mtools/mloginfo/mloginfo.py +++ b/mtools/mloginfo/mloginfo.py @@ -64,6 +64,9 @@ def run(self, arguments=None): section.run() -if __name__ == '__main__': +def main(): tool = MLogInfoTool() tool.run() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/mtools/mlogmerge/mlogmerge.py b/mtools/mlogmerge/mlogmerge.py old mode 100755 new mode 100644 index f98abd61..4903f407 --- a/mtools/mlogmerge/mlogmerge.py +++ b/mtools/mlogmerge/mlogmerge.py @@ -1,3 +1,7 @@ #!/usr/bin/env python -print "deprecated since version 1.1.0 of mtools. Use 'mlogfilter ...' instead." \ No newline at end of file +def main(): + print "deprecated since version 1.1.0 of mtools. Use 'mlogfilter ...' instead. + +if __name__ == '__main__': + sys.exit(main())" \ No newline at end of file diff --git a/mtools/mlogversion/mlogversion.py b/mtools/mlogversion/mlogversion.py old mode 100755 new mode 100644 index 98fa2d3f..bbb0d6a8 --- a/mtools/mlogversion/mlogversion.py +++ b/mtools/mlogversion/mlogversion.py @@ -1,3 +1,7 @@ #!/usr/bin/env python -print "deprecated since version 1.1.0 of mtools. Use 'mloginfo ' instead." +def main(): + print "deprecated since version 1.1.0 of mtools. Use 'mloginfo ' instead." + +if __name__ == '__main__': + sys.exit(main())" diff --git a/mtools/mlogvis/mlogvis.py b/mtools/mlogvis/mlogvis.py old mode 100755 new mode 100644 index 743dca50..654631e1 --- a/mtools/mlogvis/mlogvis.py +++ b/mtools/mlogvis/mlogvis.py @@ -84,7 +84,9 @@ def run(self, arguments=None): webbrowser.open("file://"+dstfilelocation) - -if __name__ == '__main__': +def main(): tool = MLogVisTool() tool.run() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/mtools/mplotqueries/mplotqueries.py b/mtools/mplotqueries/mplotqueries.py old mode 100755 new mode 100644 index 0d2d6ae8..28b88a5c --- a/mtools/mplotqueries/mplotqueries.py +++ b/mtools/mplotqueries/mplotqueries.py @@ -426,8 +426,9 @@ def plot(self): plt.show() -if __name__ == '__main__': +def main(): tool = MPlotQueriesTool() tool.run() - +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/mtools/util/log2code.py b/mtools/util/log2code.py old mode 100755 new mode 100644 diff --git a/scripts/mgenerate b/scripts/mgenerate deleted file mode 120000 index 8a8c2b60..00000000 --- a/scripts/mgenerate +++ /dev/null @@ -1 +0,0 @@ -../mtools/mgenerate/mgenerate.py \ No newline at end of file diff --git a/scripts/mlaunch b/scripts/mlaunch deleted file mode 120000 index 422449d2..00000000 --- a/scripts/mlaunch +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlaunch/mlaunch.py \ No newline at end of file diff --git a/scripts/mlog2json b/scripts/mlog2json deleted file mode 120000 index 9f9a7da0..00000000 --- a/scripts/mlog2json +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlog2json/mlog2json.py \ No newline at end of file diff --git a/scripts/mlogdistinct b/scripts/mlogdistinct deleted file mode 120000 index c5643a6c..00000000 --- a/scripts/mlogdistinct +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlogdistinct/mlogdistinct.py \ No newline at end of file diff --git a/scripts/mlogfilter b/scripts/mlogfilter deleted file mode 120000 index c64ec4b0..00000000 --- a/scripts/mlogfilter +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlogfilter/mlogfilter.py \ No newline at end of file diff --git a/scripts/mloginfo b/scripts/mloginfo deleted file mode 120000 index 66a672d4..00000000 --- a/scripts/mloginfo +++ /dev/null @@ -1 +0,0 @@ -../mtools/mloginfo/mloginfo.py \ No newline at end of file diff --git a/scripts/mlogmerge b/scripts/mlogmerge deleted file mode 120000 index c43ff402..00000000 --- a/scripts/mlogmerge +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlogmerge/mlogmerge.py \ No newline at end of file diff --git a/scripts/mlogversion b/scripts/mlogversion deleted file mode 120000 index b684e463..00000000 --- a/scripts/mlogversion +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlogversion/mlogversion.py \ No newline at end of file diff --git a/scripts/mlogvis b/scripts/mlogvis deleted file mode 120000 index 6e77bf92..00000000 --- a/scripts/mlogvis +++ /dev/null @@ -1 +0,0 @@ -../mtools/mlogvis/mlogvis.py \ No newline at end of file diff --git a/scripts/mplotqueries b/scripts/mplotqueries deleted file mode 120000 index 80ebd5cb..00000000 --- a/scripts/mplotqueries +++ /dev/null @@ -1 +0,0 @@ -../mtools/mplotqueries/mplotqueries.py \ No newline at end of file diff --git a/setup.py b/setup.py index abb1b298..a9010621 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,6 @@ with open('README.md') as f: long_description = f.read() - if sys.platform == 'darwin' and 'clang' in platform.python_compiler().lower(): from distutils.sysconfig import get_config_vars res = get_config_vars() @@ -79,8 +78,20 @@ package_data = { 'mtools': ['data/log2code.pickle', 'data/index.html'], }, - scripts=['scripts/mlaunch', 'scripts/mlogfilter', 'scripts/mlogvis', 'scripts/mplotqueries', 'scripts/mloginfo', \ - 'scripts/mlogversion', 'scripts/mlogmerge', 'scripts/mlog2json', 'scripts/mlogdistinct', 'scripts/mgenerate'], + entry_points={ + "console_scripts": [ + "mgenerate=mtools.mgenerate.mgenerate:main", + "mlaunch=mtools.mlaunch.mlaunch:main", + "mlog2json=mtools.mlog2json.mlog2json:main", + "mlogdistinct=mtools.mlogdistinct.mlogdistinct:main", + "mlogfilter=mtools.mlogfilter.mlogfilter:main", + "mloginfo=mtools.mloginfo.mloginfo:main", + "mlogmerge=mtools.mlogmerge.mlogmerge:main", + "mlogversion=mtools.mlogversion.mlogversion:main", + "mlogvis=mtools.mlogvis.mlogvis:main", + "mplotqueries=mtools.mplotqueries.mplotqueries:main" + ], + }, author='Thomas Rueckstiess', author_email='thomas@rueckstiess.net', url='/~https://github.com/rueckstiess/mtools', From 58833fc42ad4e5c6fd211b6c01bed0c51740227f Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Sat, 20 Dec 2014 18:29:37 +1100 Subject: [PATCH 02/22] bumped to 1.1.8-dev --- CHANGES.md | 3 +++ README.md | 2 +- mtools/version.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5a094b72..8adb6f9e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ Changes to mtools ================= +#### version 1.1.8 + + #### version 1.1.7 * mtools now understands 2.8 style log format, with severity and components. Added by @jimoleary (#269) diff --git a/README.md b/README.md index 9bf37049..84a570e5 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ instructions for these modules. Recent Changes -------------- -The current version of mtools is 1.1.7. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools. +The current version of mtools is 1.1.8-dev. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools. Contribute to mtools diff --git a/mtools/version.py b/mtools/version.py index 9910ac22..003f37d1 100644 --- a/mtools/version.py +++ b/mtools/version.py @@ -1 +1 @@ -__version__ = '1.1.7' +__version__ = '1.1.8-dev' From b21433bca08f64c315de7d4273021ae736d4b617 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Wed, 7 Jan 2015 13:07:53 +1100 Subject: [PATCH 03/22] updated log component strings. Fixes #327. --- mtools/util/logevent.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/mtools/util/logevent.py b/mtools/util/logevent.py index 0c124ea6..5db1af7b 100644 --- a/mtools/util/logevent.py +++ b/mtools/util/logevent.py @@ -50,19 +50,9 @@ class LogEvent(object): log_operations = ['query', 'insert', 'update', 'remove', 'getmore', 'command'] log_levels = ['D', 'F', 'E', 'W', 'I', 'U'] - log_components = ['-', - 'ACCESS', - 'COMMANDS', - 'INDEXING', - 'NETWORK', - 'QUERY', - 'REPLSETS', - 'SHARDING', - 'STORAGE', - 'JOURNAL', - 'WRITES', - 'S2', - 'TOTAL'] + log_components = ['-','ACCESS','COMMAND','CONTROL','GEO', \ + 'INDEX','NETWORK','QUERY','REPL','SHARDING', \ + 'STORAGE','JOURNAL','WRITE','TOTAL'] def __init__(self, doc_or_str): self._year_rollover = False From 5adfa18fa3a08ee8e51eeec67e45a15e1e77165b Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Wed, 7 Jan 2015 17:28:09 +1100 Subject: [PATCH 04/22] fixed tests for component change and year rollover *sigh*. --- mtools/test/logfiles/mongod_278.log | 10 +++++----- mtools/test/logfiles/mongod_278_partial.log | 10 +++++----- mtools/test/test_mlogfilter.py | 16 ++++++++-------- mtools/test/test_util_logfile.py | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mtools/test/logfiles/mongod_278.log b/mtools/test/logfiles/mongod_278.log index 857d3f0e..01e228e6 100644 --- a/mtools/test/logfiles/mongod_278.log +++ b/mtools/test/logfiles/mongod_278.log @@ -10,16 +10,16 @@ 2014-10-31T13:00:03.916+0000 I - [initandlisten] options: { net: { port: 27017 }, processManagement: { fork: true }, storage: { dbPath: "/Users/joleary/Documents/Support/CS-16129/data/db" }, systemLog: { destination: "file", logAppend: true, path: "/Users/joleary/Documents/Support/CS-16129/data/mongod.log" } } 2014-10-31T13:00:03.917+0000 D ACCESS [initandlisten] journal dir=/Users/joleary/Documents/Support/CS-16129/data/db/journal 2014-10-31T13:00:03.917+0000 E JOURNAL [initandlisten] recover : no journal files present, no recovery needed -2014-10-31T13:00:03.930+0000 F INDEXING [initandlisten] allocating new ns file /Users/joleary/Documents/Support/CS-16129/data/db/local.ns, filling with zeroes... +2014-10-31T13:00:03.930+0000 F INDEX [initandlisten] allocating new ns file /Users/joleary/Documents/Support/CS-16129/data/db/local.ns, filling with zeroes... 2014-10-31T13:00:03.996+0000 U STORAGE [FileAllocator] allocating new datafile /Users/joleary/Documents/Support/CS-16129/data/db/local.0, filling with zeroes... 2014-10-31T13:00:03.996+0000 W STORAGE [FileAllocator] creating directory /Users/joleary/Documents/Support/CS-16129/data/db/_tmp 2014-10-31T13:00:04.258+0000 I STORAGE [FileAllocator] done allocating datafile /Users/joleary/Documents/Support/CS-16129/data/db/local.0, size: 64MB, took 0.261 secs 2014-10-31T13:00:04.295+0000 I QUERY [initandlisten] command local.$cmd command: create { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 numYields:0 reslen:37 365ms -2014-10-31T13:00:04.295+0000 I COMMANDS [initandlisten] waiting for connections on port 27017 -2014-10-31T13:00:04.458+0000 I REPLSETS [initandlisten] connection accepted from 127.0.0.1:51869 #1 (1 connection now open) +2014-10-31T13:00:04.295+0000 I COMMAND [initandlisten] waiting for connections on port 27017 +2014-10-31T13:00:04.458+0000 I REPL [initandlisten] connection accepted from 127.0.0.1:51869 #1 (1 connection now open) 2014-10-31T13:00:04.459+0000 I SHARDING [conn1] end connection 127.0.0.1:51869 (0 connections now open) 2014-10-31T13:00:04.460+0000 I STORAGE [initandlisten] connection accepted from 127.0.0.1:51870 #2 (1 connection now open) 2014-10-31T13:00:04.461+0000 I NETWORK [conn2] end connection 127.0.0.1:51870 (0 connections now open) -2014-10-31T13:00:04.461+0000 I WRITES [conn2] end connection 127.0.0.1:51870 (0 connections now open) -2014-10-31T13:00:04.461+0000 I S2 [conn2] end connection 127.0.0.1:51870 (0 connections now open) +2014-10-31T13:00:04.461+0000 I WRITE [conn2] end connection 127.0.0.1:51870 (0 connections now open) +2014-10-31T13:00:04.461+0000 I GEO [conn2] end connection 127.0.0.1:51870 (0 connections now open) 2014-10-31T13:00:04.461+0000 I TOTAL [conn2] end connection 127.0.0.1:51870 (0 connections now open) \ No newline at end of file diff --git a/mtools/test/logfiles/mongod_278_partial.log b/mtools/test/logfiles/mongod_278_partial.log index 7160b65d..dc218fc3 100644 --- a/mtools/test/logfiles/mongod_278_partial.log +++ b/mtools/test/logfiles/mongod_278_partial.log @@ -4,16 +4,16 @@ 2014-10-31T13:00:03.916+0000 I - [initandlisten] options: { net: { port: 27017 }, processManagement: { fork: true }, storage: { dbPath: "/Users/joleary/Documents/Support/CS-16129/data/db" }, systemLog: { destination: "file", logAppend: true, path: "/Users/joleary/Documents/Support/CS-16129/data/mongod.log" } } 2014-10-31T13:00:03.917+0000 D ACCESS [initandlisten] journal dir=/Users/joleary/Documents/Support/CS-16129/data/db/journal 2014-10-31T13:00:03.917+0000 E JOURNAL [initandlisten] recover : no journal files present, no recovery needed -2014-10-31T13:00:03.930+0000 F INDEXING [initandlisten] allocating new ns file /Users/joleary/Documents/Support/CS-16129/data/db/local.ns, filling with zeroes... +2014-10-31T13:00:03.930+0000 F INDEX [initandlisten] allocating new ns file /Users/joleary/Documents/Support/CS-16129/data/db/local.ns, filling with zeroes... 2014-10-31T13:00:03.996+0000 U STORAGE [FileAllocator] allocating new datafile /Users/joleary/Documents/Support/CS-16129/data/db/local.0, filling with zeroes... 2014-10-31T13:00:03.996+0000 W STORAGE [FileAllocator] creating directory /Users/joleary/Documents/Support/CS-16129/data/db/_tmp 2014-10-31T13:00:04.258+0000 I STORAGE [FileAllocator] done allocating datafile /Users/joleary/Documents/Support/CS-16129/data/db/local.0, size: 64MB, took 0.261 secs 2014-10-31T13:00:04.295+0000 I QUERY [initandlisten] command local.$cmd command: create { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 numYields:0 reslen:37 365ms -2014-10-31T13:00:04.295+0000 I COMMANDS [initandlisten] waiting for connections on port 27017 -2014-10-31T13:00:04.458+0000 I REPLSETS [initandlisten] connection accepted from 127.0.0.1:51869 #1 (1 connection now open) +2014-10-31T13:00:04.295+0000 I COMMAND [initandlisten] waiting for connections on port 27017 +2014-10-31T13:00:04.458+0000 I REPL [initandlisten] connection accepted from 127.0.0.1:51869 #1 (1 connection now open) 2014-10-31T13:00:04.459+0000 I SHARDING [conn1] end connection 127.0.0.1:51869 (0 connections now open) 2014-10-31T13:00:04.460+0000 I STORAGE [initandlisten] connection accepted from 127.0.0.1:51870 #2 (1 connection now open) 2014-10-31T13:00:04.461+0000 I NETWORK [conn2] end connection 127.0.0.1:51870 (0 connections now open) -2014-10-31T13:00:04.461+0000 I WRITES [conn2] end connection 127.0.0.1:51870 (0 connections now open) -2014-10-31T13:00:04.461+0000 I S2 [conn2] end connection 127.0.0.1:51870 (0 connections now open) +2014-10-31T13:00:04.461+0000 I WRITE [conn2] end connection 127.0.0.1:51870 (0 connections now open) +2014-10-31T13:00:04.461+0000 I GEO [conn2] end connection 127.0.0.1:51870 (0 connections now open) 2014-10-31T13:00:04.461+0000 I TOTAL [conn2] end connection 127.0.0.1:51870 (0 connections now open) \ No newline at end of file diff --git a/mtools/test/test_mlogfilter.py b/mtools/test/test_mlogfilter.py index a3fcb979..7cd1a351 100644 --- a/mtools/test/test_mlogfilter.py +++ b/mtools/test/test_mlogfilter.py @@ -398,10 +398,10 @@ def test_year_rollover_1(self): # load year rollover logfile yro_logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'year_rollover.log') - self.tool.run('%s --from Jan 1 2014 --timestamp-format iso8601-utc' % yro_logfile_path) + self.tool.run('%s --from Jan 1 2015 --timestamp-format iso8601-utc' % yro_logfile_path) output = sys.stdout.getvalue() for line in output.splitlines(): - assert line.startswith("2014-") + assert line.startswith("2015-") def test_year_rollover_2(self): @@ -414,7 +414,7 @@ def test_year_rollover_2(self): output = sys.stdout.getvalue() assert len(output.splitlines()) > 0 for line in output.splitlines(): - assert line.startswith("2013-") + assert line.startswith("2014-") def test_level_225(self): """ mlogfilter: test that mlogfilter works levels on older logs """ @@ -464,19 +464,19 @@ def test_method(self): setattr(cls,test_method.__name__,test_method) _add_component_test(TestMLogFilter, 'test_total_component', 'TOTAL', 1) -_add_component_test(TestMLogFilter, 'test_s2_component', 'S2', 1) +_add_component_test(TestMLogFilter, 'test_s2_component', 'GEO', 1) _add_component_test(TestMLogFilter, 'test_all_component', " ".join(LogEvent.log_components), 25) _add_component_test(TestMLogFilter, 'test_dash_component', '-', 10) _add_component_test(TestMLogFilter, 'test_access_component', 'ACCESS', 1) -_add_component_test(TestMLogFilter, 'test_commands_component', 'COMMANDS', 1) -_add_component_test(TestMLogFilter, 'test_indexing_component', 'INDEXING', 1) +_add_component_test(TestMLogFilter, 'test_commands_component', 'COMMAND', 1) +_add_component_test(TestMLogFilter, 'test_indexing_component', 'INDEX', 1) _add_component_test(TestMLogFilter, 'test_network_component', 'NETWORK', 1) _add_component_test(TestMLogFilter, 'test_query_component', 'QUERY', 1) -_add_component_test(TestMLogFilter, 'test_replsets_component', 'REPLSETS', 1) +_add_component_test(TestMLogFilter, 'test_replsets_component', 'REPL', 1) _add_component_test(TestMLogFilter, 'test_sharding_component', 'SHARDING', 1) _add_component_test(TestMLogFilter, 'test_storage_component', 'STORAGE', 4) _add_component_test(TestMLogFilter, 'test_journal_component', 'JOURNAL', 1) -_add_component_test(TestMLogFilter, 'test_writes_component', 'WRITES', 1) +_add_component_test(TestMLogFilter, 'test_writes_component', 'WRITE', 1) def _add_level_test(cls, name, level, expected=1): diff --git a/mtools/test/test_util_logfile.py b/mtools/test/test_util_logfile.py index 6402842b..58b26e81 100644 --- a/mtools/test/test_util_logfile.py +++ b/mtools/test/test_util_logfile.py @@ -37,8 +37,8 @@ def test_start_end(self): logfile = LogFile(self.file_year_rollover) - assert logfile.start == datetime(2013, 12, 30, 00, 13, 01, 661000, tzutc()) - assert logfile.end == datetime(2014, 01, 02, 23, 27, 11, 720000, tzutc()) + assert logfile.start == datetime(2014, 12, 30, 00, 13, 01, 661000, tzutc()) + assert logfile.end == datetime(2015, 01, 02, 23, 27, 11, 720000, tzutc()) def test_timezone(self): From a333fc2299e30a839b77a7bdd282348395417df7 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 22 Jan 2015 11:55:26 +1100 Subject: [PATCH 05/22] added --wiredTigerEngineConfigString to the whitelisted arguments passed to mongod. --- mtools/mlaunch/mlaunch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py index 80be004d..81290dde 100755 --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -992,6 +992,10 @@ def _filter_valid_arguments(self, arguments, binary="mongod", config=False): continue accepted_arguments.append(argument) + # add undocumented option + if binary == "mongod": + accepted_arguments.append('--wiredTigerEngineConfigString') + # filter valid arguments result = [] for i, arg in enumerate(arguments): From cb686a98671f88306fddb3ecd7318a894b909c4b Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 12 Feb 2015 12:31:09 +1100 Subject: [PATCH 06/22] added writeConflicts property to LogEvent. Fixes #334. --- mtools/util/logevent.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mtools/util/logevent.py b/mtools/util/logevent.py index 5db1af7b..ef100879 100644 --- a/mtools/util/logevent.py +++ b/mtools/util/logevent.py @@ -106,6 +106,7 @@ def _reset(self): self._ndeleted = None self._numYields = None self._planSummary = None + self._writeConflicts = None self._r = None self._w = None self._conn = None @@ -449,6 +450,17 @@ def ntoreturn(self): return self._ntoreturn + @property + def writeConflicts(self): + """ extract ntoreturn counter if available (lazy) """ + + if not self._counters_calculated: + self._counters_calculated = True + self._extract_counters() + + return self._writeConflicts + + @property def nreturned(self): """ extract nreturned counter if available (lazy) """ @@ -538,7 +550,7 @@ def _extract_counters(self): # extract counters (if present) counters = ['nscanned', 'ntoreturn', 'nreturned', 'ninserted', \ - 'nupdated', 'ndeleted', 'r', 'w', 'numYields', 'planSummary'] + 'nupdated', 'ndeleted', 'r', 'w', 'numYields', 'planSummary', 'writeConflicts', 'keyUpdates'] split_tokens = self.split_tokens From a689a8fb594e6ee2fd0822754779ceec05c55a0b Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 12 Feb 2015 12:33:54 +1100 Subject: [PATCH 07/22] change .command to always be lowercase. Fixes #335. --- mtools/util/logevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtools/util/logevent.py b/mtools/util/logevent.py index ef100879..186ff4c2 100644 --- a/mtools/util/logevent.py +++ b/mtools/util/logevent.py @@ -423,7 +423,7 @@ def command(self): if command == '{': # workaround for <= 2.2 log files, where command was not listed separately command = self.split_tokens[command_idx+2][:-1] - self._command = command + self._command = command.lower() except ValueError: pass From c8e622c0d00251b4a00c679668d341776ca0b2d3 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 12 Feb 2015 12:56:55 +1100 Subject: [PATCH 08/22] skip truncation warnings and try to parse as much as possible from line. Fixes #133. --- mtools/test/test_util_logevent.py | 14 +++++++++++--- mtools/util/logevent.py | 19 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/mtools/test/test_util_logevent.py b/mtools/test/test_util_logevent.py index f029468d..df2e139d 100644 --- a/mtools/test/test_util_logevent.py +++ b/mtools/test/test_util_logevent.py @@ -18,6 +18,7 @@ line_pattern_26_c = """2014-03-18T18:34:50.777+1100 [conn10] query test.new query: { $query: { a: 1.0 }, $orderby: { b: 1.0 } } planSummary: EOF ntoreturn:0 ntoskip:0 keyUpdates:0 numYields:0 locks(micros) r:60 nreturned:0 reslen:20 0ms""" line_command_26_a = """2014-11-21T18:20:20.263+0800 [conn9] command admin.$cmd command: replSetGetStatus { replSetGetStatus: 1.0, forShell: 1.0 } keyUpdates:0 numYields:0 reslen:76 0ms""" line_command_26_b = """2014-11-21T18:20:57.076+0800 [conn9] command test.$cmd command: aggregate { aggregate: "mgendata", pipeline: [ { $group: { _id: "$bar", avg: { $avg: "$foo" } } } ], cursor: {} } keyUpdates:0 numYields:0 locks(micros) r:6783 reslen:229 11ms""" +line_truncated_24 = """Wed Jan 28 00:31:16.302 [conn12345] warning: log line attempted (26k) over max size(10k), printing beginning and end ... getmore MyDB.MyColl query: { foo: ObjectId('123456789012345678901234'), foo: { $in: [ 1, 2, 3 ] }, bar: false } cursorid:1234567890123456789 ntoreturn:0 keyUpdates:0 numYields: 23 locks(micros) r:24715 nreturned:1324 reslen:256993 1445ms""" # fake system.profile documents profile_doc1 = { "op" : "query", "ns" : "test.foo", "thread": "test.system.profile", "query" : { "test" : 1 }, "ntoreturn" : 0, "ntoskip" : 0, "nscanned" : 0, "keyUpdates" : 0, "numYield" : 0, "lockStats" : { "timeLockedMicros" : { "r" : 461, "w" :0 }, "timeAcquiringMicros" : { "r" : 4, "w" : 3 } }, "nreturned" : 0, "responseLength" : 20, "millis" : 0, "ts" : parser.parse("2014-03-20T04:04:21.231Z"), "client" : "127.0.0.1", "allUsers" : [ ], "user" : "" } @@ -77,7 +78,7 @@ def test_logevent_pattern_parsing(): def test_logevent_command_parsing(): le = LogEvent(line_command_26_a) - assert(le.command) == 'replSetGetStatus' + assert(le.command) == 'replsetgetstatus' le = LogEvent(line_command_26_b) assert(le.command) == 'aggregate' @@ -121,11 +122,18 @@ def test_logevent_profile_sort_pattern_parsing(): def test_logevent_extract_new_and_old_numYields(): - le = LogEvent(line_246_numYields) + le = LogEvent(line_246_numYields) assert(le.numYields == 2405) - le = LogEvent(line_253_numYields) + le = LogEvent(line_253_numYields) assert(le.numYields == 1) + + +def test_logevent_parse_truncated_line(): + le = LogEvent(line_truncated_24) + assert(le.thread == "conn12345") + assert(le.numYields == 23) + assert(le.operation == "getmore") def test_logevent_extract_planSummary(): diff --git a/mtools/util/logevent.py b/mtools/util/logevent.py index 186ff4c2..e2202b54 100644 --- a/mtools/util/logevent.py +++ b/mtools/util/logevent.py @@ -369,14 +369,25 @@ def _extract_operation_and_namespace(self): split_tokens = self.split_tokens if not self._datetime_nextpos: - # force evaluation of datetime to get access to datetime_offset - _ = self.datetime + # force evaluation of thread to get access to datetime_offset and to + # protect from changes due to line truncation + _ = self.thread if not self._datetime_nextpos or len(split_tokens) <= self._datetime_nextpos + 2: return op = split_tokens[self._datetime_nextpos + 1] + if op == 'warning:': + # check if this log line got truncated + if ("warning: log line attempted" in self._line_str) and \ + ("over max size") in self._line_str: + self._datetime_nextpos = split_tokens.index('...') + op = split_tokens[self._datetime_nextpos + 1] + else: + # unknown warning, bail out + return + if op in self.log_operations: self._operation = op self._namespace = split_tokens[self._datetime_nextpos + 2] @@ -554,8 +565,8 @@ def _extract_counters(self): split_tokens = self.split_tokens - # trigger thread evaluation to get access to offset - if self.thread: + # trigger operation evaluation to get access to offset + if self.operation: for t, token in enumerate(split_tokens[self.datetime_nextpos+2:]): for counter in counters: if token.startswith('%s:'%counter): From 2b2f9fb037b28858c0a71766b0e65897b528054f Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 12 Feb 2015 15:04:22 +1100 Subject: [PATCH 09/22] removed deprecated scripts mlog2json, mlogdistinct, mlogmerge, mlogversion. --- mtools/mlog2json/__init__.py | 0 mtools/mlog2json/mlog2json.py | 7 ------- mtools/mlogdistinct/__init__.py | 0 mtools/mlogdistinct/mlogdistinct.py | 7 ------- mtools/mlogmerge/__init__.py | 0 mtools/mlogmerge/mlogmerge.py | 7 ------- mtools/mlogversion/__init__.py | 0 mtools/mlogversion/mlogversion.py | 7 ------- setup.py | 8 -------- 9 files changed, 36 deletions(-) delete mode 100644 mtools/mlog2json/__init__.py delete mode 100644 mtools/mlog2json/mlog2json.py delete mode 100644 mtools/mlogdistinct/__init__.py delete mode 100644 mtools/mlogdistinct/mlogdistinct.py delete mode 100644 mtools/mlogmerge/__init__.py delete mode 100644 mtools/mlogmerge/mlogmerge.py delete mode 100644 mtools/mlogversion/__init__.py delete mode 100644 mtools/mlogversion/mlogversion.py diff --git a/mtools/mlog2json/__init__.py b/mtools/mlog2json/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mtools/mlog2json/mlog2json.py b/mtools/mlog2json/mlog2json.py deleted file mode 100644 index e6bc39b1..00000000 --- a/mtools/mlog2json/mlog2json.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -def main(): - print "deprecated since version 1.1.0 of mtools. Use 'mlogfilter --json' instead." - -if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file diff --git a/mtools/mlogdistinct/__init__.py b/mtools/mlogdistinct/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mtools/mlogdistinct/mlogdistinct.py b/mtools/mlogdistinct/mlogdistinct.py deleted file mode 100644 index da54d04e..00000000 --- a/mtools/mlogdistinct/mlogdistinct.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -def main(): - print "deprecated since version 1.1.0 of mtools. Use 'mloginfo --distinct' instead." - -if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file diff --git a/mtools/mlogmerge/__init__.py b/mtools/mlogmerge/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mtools/mlogmerge/mlogmerge.py b/mtools/mlogmerge/mlogmerge.py deleted file mode 100644 index 4903f407..00000000 --- a/mtools/mlogmerge/mlogmerge.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -def main(): - print "deprecated since version 1.1.0 of mtools. Use 'mlogfilter ...' instead. - -if __name__ == '__main__': - sys.exit(main())" \ No newline at end of file diff --git a/mtools/mlogversion/__init__.py b/mtools/mlogversion/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mtools/mlogversion/mlogversion.py b/mtools/mlogversion/mlogversion.py deleted file mode 100644 index bbb0d6a8..00000000 --- a/mtools/mlogversion/mlogversion.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -def main(): - print "deprecated since version 1.1.0 of mtools. Use 'mloginfo ' instead." - -if __name__ == '__main__': - sys.exit(main())" diff --git a/setup.py b/setup.py index 26844769..36e83b34 100644 --- a/setup.py +++ b/setup.py @@ -43,10 +43,6 @@ 'mtools.mloginfo', 'mtools.mlogvis', 'mtools.mplotqueries', - 'mtools.mlogversion', - 'mtools.mlogdistinct', - 'mtools.mlogmerge', - 'mtools.mlog2json', 'mtools.mgenerate', 'mtools.test', 'mtools.util', @@ -83,12 +79,8 @@ "console_scripts": [ "mgenerate=mtools.mgenerate.mgenerate:main", "mlaunch=mtools.mlaunch.mlaunch:main", - "mlog2json=mtools.mlog2json.mlog2json:main", - "mlogdistinct=mtools.mlogdistinct.mlogdistinct:main", "mlogfilter=mtools.mlogfilter.mlogfilter:main", "mloginfo=mtools.mloginfo.mloginfo:main", - "mlogmerge=mtools.mlogmerge.mlogmerge:main", - "mlogversion=mtools.mlogversion.mlogversion:main", "mlogvis=mtools.mlogvis.mlogvis:main", "mplotqueries=mtools.mplotqueries.mplotqueries:main" ], From 0a1b4dae5ed30763a3acae32457f27aa6792ae9a Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 12 Feb 2015 15:07:10 +1100 Subject: [PATCH 10/22] The import of ProfileCollection needs to happen inside of the outer try: statement for a fallback to simple argparse.FileType for InputSourceAction. Fixes #337. --- mtools/util/cmdlinetool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mtools/util/cmdlinetool.py b/mtools/util/cmdlinetool.py index 28b485a9..6d9d0fdc 100644 --- a/mtools/util/cmdlinetool.py +++ b/mtools/util/cmdlinetool.py @@ -8,7 +8,6 @@ from dateutil.tz import tzutc from mtools.version import __version__ -from mtools.util.profile_collection import ProfileCollection from mtools.util.logfile import LogFile try: @@ -18,6 +17,8 @@ from pymongo import Connection from pymongo.errors import ConnectionFailure, AutoReconnect, OperationFailure, ConfigurationError + from mtools.util.profile_collection import ProfileCollection + class InputSourceAction(argparse.FileType): """ This class extends the FileType class from the argparse module. It will try to open the file and pass the handle to a new LogFile object, but if that's not possible it From 295479b2dfe5310139bba4959bd70bf1868586f9 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Thu, 12 Feb 2015 15:23:24 +1100 Subject: [PATCH 11/22] added numpy minimum version (1.8.0) in INSTALL.md. Fixes #332. --- INSTALL.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 2307f3f6..c77c70a8 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -104,8 +104,7 @@ Installation instructions for matplotlib can be found under the [matplotlib Inst *required for matplotlib (in mplotqueries)* -[NumPy](http://numpy.scipy.org/) is a Python module for scientific computing and numerical calculations. -Try installing NumPy with pip, by doing: +[NumPy](http://numpy.scipy.org/) is a Python module for scientific computing and numerical calculations. Version 1.8.0 or higher is required for mtools. Try installing NumPy with pip, by doing: pip install numpy From 3adb9576cd572163a6f30810a62edc330494c998 Mon Sep 17 00:00:00 2001 From: Gianfranco Palumbo Date: Mon, 9 Mar 2015 17:46:38 +0000 Subject: [PATCH 12/22] Fix $missing operator option --- mtools/mgenerate/mgenerate_spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtools/mgenerate/mgenerate_spec.md b/mtools/mgenerate/mgenerate_spec.md index a0d7ec9a..d234babd 100644 --- a/mtools/mgenerate/mgenerate_spec.md +++ b/mtools/mgenerate/mgenerate_spec.md @@ -121,7 +121,7 @@ Will not insert the key/value pair. A percentage of missing values can be specif ###### Missing Percentage -`{ "$missing" : { "percentage" : 30, "ifnot" : VALUE } }`
+`{ "$missing" : { "percent" : 30, "ifnot" : VALUE } }`
Will cause the key/value pair to be missing 30% of the time, and otherwise set the VALUE for the given key. From 30e62f8ef8640b2bb1f7840e905cc0d60fad1946 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 13:32:37 +1100 Subject: [PATCH 13/22] added $geo alias for mgenerate coordinates --- mtools/mgenerate/operators/operators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtools/mgenerate/operators/operators.py b/mtools/mgenerate/operators/operators.py index a044a0c9..d64a5236 100644 --- a/mtools/mgenerate/operators/operators.py +++ b/mtools/mgenerate/operators/operators.py @@ -203,7 +203,7 @@ class CoordinateOperator(BaseOperator): dict_format = True string_format = True - names = ['$coordinates', '$coordinate', '$coord'] + names = ['$coordinates', '$coordinate', '$coord', '$geo'] defaults = OrderedDict([ ('long_lim', [-180, 180]), ('lat_lim', [-90, 90]) ]) def __call__(self, options=None): From 5675455a4532c85eb8555b27b0704ddf3319c9c0 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 13:33:01 +1100 Subject: [PATCH 14/22] fixed mlaunch sharded cluster with auth for 3.0 --- mtools/mlaunch/mlaunch.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py index 7300b662..fde23917 100644 --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -92,6 +92,10 @@ def shutdown_host(port, username=None, password=None, authdb=None): mc.admin.command('shutdown', force=True) except AutoReconnect: pass + except OperationFailure: + print "Error: cannot authenticate to shut down %s." % host + return + except ConnectionFailure: pass else: @@ -237,9 +241,11 @@ def run(self, arguments=None): # branch out in sub-commands getattr(self, self.args['command'])() + def _read_key_file(): + with open(os.path.join(self.dir, 'keyfile'), 'r') as f: + return ''.join(f.readlines()) # -- below are the main commands: init, start, stop, list, kill - def init(self): """ sub-command init. Branches out to sharded, replicaset or single node methods. """ @@ -278,7 +284,7 @@ def init(self): # start mongod (shard and config) nodes and wait nodes = self.get_tagged(['mongod', 'down']) - self._start_on_ports(nodes, wait=True) + self._start_on_ports(nodes, wait=True, overrideAuth=True) # initiate replica sets if init is called for the first time if first_init: @@ -289,7 +295,7 @@ def init(self): # add mongos mongos = sorted(self.get_tagged(['mongos', 'down'])) - self._start_on_ports(mongos, wait=True) + self._start_on_ports(mongos, wait=True, overrideAuth=True) if first_init: # add shards @@ -331,8 +337,8 @@ def init(self): print res, '- will retry' time.sleep(1) - + elif self.args['single']: # just start node nodes = self.get_tagged(['single', 'down']) @@ -373,7 +379,7 @@ def init(self): raise RuntimeError("can't connect to server, so adding admin user isn't possible") if "clusterAdmin" not in self.args['auth_roles']: - warnings.warn("the stop command will not work with auth if the user does not have the clusterAdmin role") + warnings.warn("the stop command will not work with auth because the user does not have the clusterAdmin role") self._add_user(sorted(nodes)[0], name=self.args['username'], password=self.args['password'], database=self.args['auth_db'], roles=self.args['auth_roles']) @@ -391,7 +397,6 @@ def init(self): authdb = self.args['auth_db'] if self.args['auth'] else None shutdown_host(port, username, password, authdb) - # write out parameters if self.args['verbose']: print "writing .mlaunch_startup file." @@ -400,6 +405,12 @@ def init(self): # discover again, to get up-to-date info self.discover() + # for sharded authenticated clusters, restart after first_init to enable auth + if self.args['sharded'] and self.args['auth'] and first_init: + if self.args['verbose']: + print "restarting cluster to enable auth..." + self.restart() + if self.args['verbose']: print "done." @@ -939,6 +950,8 @@ def _create_paths(self, basedir, name=None): def _get_ports_from_args(self, args, extra_tag): tags = [] + if 'tags' not in args: + args['tags'] = [] for tag1, tag2 in zip(args['tags'][:-1], args['tags'][1:]): if re.match('^\d{1,2}$', tag1): @@ -1033,12 +1046,19 @@ def _get_shard_names(self, args): return shard_names - - def _start_on_ports(self, ports, wait=False): + def _start_on_ports(self, ports, wait=False, overrideAuth=False): threads = [] + if overrideAuth and self.args['verbose']: + print "creating cluster without auth for setup, will enable auth at the end..." + for port in ports: command_str = self.startup_info[str(port)] + + if overrideAuth: + # this is to set up sharded clusters without auth first, then relaunch with auth + command_str = re.sub(r'--keyFile \S+', '', command_str) + ret = subprocess.call([command_str], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) binary = command_str.split()[0] From 2ac8f60c48d8bbaac09ea5f2cb883b457b1dc555 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 13:47:53 +1100 Subject: [PATCH 15/22] updated to psutil>=2.0 p.name() and p.cmdline() are now methods --- mtools/mlaunch/mlaunch.py | 12 +++++++----- requirements.txt | 2 +- setup.py | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py index fde23917..c48eece6 100644 --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -241,9 +241,6 @@ def run(self, arguments=None): # branch out in sub-commands getattr(self, self.args['command'])() - def _read_key_file(): - with open(os.path.join(self.dir, 'keyfile'), 'r') as f: - return ''.join(f.readlines()) # -- below are the main commands: init, start, stop, list, kill def init(self): @@ -1121,7 +1118,7 @@ def _get_processes(self): for p in psutil.process_iter(): # skip all but mongod / mongos - if p.name not in ['mongos', 'mongod']: + if p.name() not in ['mongos', 'mongod']: continue port = None @@ -1129,7 +1126,7 @@ def _get_processes(self): # compare ports based on command line argument startup = self.startup_info[possible_port].split() try: - p_port = p.cmdline[p.cmdline.index('--port')+1] + p_port = p.cmdline()[p.cmdline().index('--port')+1] startup_port = startup[startup.index('--port')+1] except ValueError: continue @@ -1316,6 +1313,11 @@ def _construct_mongos(self, logpath, port, configdb): # store parameters in startup_info self.startup_info[str(port)] = command_str + + def _read_key_file(): + with open(os.path.join(self.dir, 'keyfile'), 'r') as f: + return ''.join(f.readlines()) + def main(): tool = MLaunchTool() diff --git a/requirements.txt b/requirements.txt index 85405494..2a54f709 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ matplotlib==1.3.1 nose==1.3.0 numpy==1.8.0 pymongo==2.6.3 -psutil==1.2.1 +psutil>=2.0 diff --git a/setup.py b/setup.py index 36e83b34..80d32ca7 100644 --- a/setup.py +++ b/setup.py @@ -10,8 +10,8 @@ from setuptools import setup, find_packages # test for 2.7-included packages, add to requirements if not available - install_requires = ['psutil'] - test_requires = ['nose>=1.0', 'psutil', 'pymongo>=2.4'] + install_requires = ['psutil>=2.0'] + test_requires = ['nose>=1.0', 'psutil>=2.0', 'pymongo>=2.4'] try: import argparse except ImportError: From 57efc47a8545c69112587454523a710bace3dbf6 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 14:15:35 +1100 Subject: [PATCH 16/22] aliased --sharded with --shards --- mtools/mlaunch/mlaunch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py index c48eece6..4bc51a39 100644 --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -166,7 +166,7 @@ def run(self, arguments=None): init_parser.add_argument('--name', action='store', metavar='NAME', default='replset', help='name for replica set (default=replset)') # sharded clusters - init_parser.add_argument('--sharded', action='store', nargs='+', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards.') + init_parser.add_argument('--sharded', '--shards', action='store', nargs='+', metavar='N', help='creates a sharded setup consisting of several singles or replica sets. Provide either list of shard names or number of shards.') init_parser.add_argument('--config', action='store', default=1, type=int, metavar='NUM', choices=[1, 3], help='adds NUM config servers to sharded setup (requires --sharded, NUM must be 1 or 3, default=1)') init_parser.add_argument('--mongos', action='store', default=1, type=int, metavar='NUM', help='starts NUM mongos processes (requires --sharded, default=1)') From 032a3a3585c9fabd2bc9635249831636568097a2 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 14:36:16 +1100 Subject: [PATCH 17/22] added storage engine information to log file and mloginfo --- mtools/mloginfo/mloginfo.py | 7 +++---- mtools/test/logfiles/wiredtiger.log | 26 ++++++++++++++++++++++++++ mtools/test/test_util_logfile.py | 17 +++++++++++++++++ mtools/util/logfile.py | 22 +++++++++++++++++++--- 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 mtools/test/logfiles/wiredtiger.log diff --git a/mtools/mloginfo/mloginfo.py b/mtools/mloginfo/mloginfo.py index d4920908..716b17a3 100644 --- a/mtools/mloginfo/mloginfo.py +++ b/mtools/mloginfo/mloginfo.py @@ -51,7 +51,6 @@ def run(self, arguments=None): print " length: %s" % len(self.logfile) print " binary: %s" % (self.logfile.binary or "unknown") - version = (' -> '.join(self.logfile.versions) or "unknown") # if version is unknown, go by date @@ -63,12 +62,12 @@ def run(self, arguments=None): elif self.logfile.datetime_format == "iso8601-utc" or \ self.logfile.datetime_format == "iso8601-local": if self.logfile.has_level: - version = '>= 2.8 (iso8601 format, level, component)' + version = '>= 3.0 (iso8601 format, level, component)' else: version = '= 2.6.x (iso8601 format)' - print " version: %s" % version, - print + print " version: %s" % version + print " storage: %s" % (self.logfile.storage_engine or 'unknown') # now run all sections for section in self.sections: diff --git a/mtools/test/logfiles/wiredtiger.log b/mtools/test/logfiles/wiredtiger.log new file mode 100644 index 00000000..9ff45d93 --- /dev/null +++ b/mtools/test/logfiles/wiredtiger.log @@ -0,0 +1,26 @@ +2015-03-10T14:21:24.484+1100 I CONTROL [initandlisten] MongoDB starting : pid=36239 port=27017 dbpath=/Users/tr/Documents/tmp/data/db 64-bit host=enter.local +2015-03-10T14:21:24.485+1100 I CONTROL [initandlisten] db version v3.0.0 +2015-03-10T14:21:24.485+1100 I CONTROL [initandlisten] git version: a841fd6394365954886924a35076691b4d149168 +2015-03-10T14:21:24.485+1100 I CONTROL [initandlisten] build info: Darwin bs-osx108-1 12.5.0 Darwin Kernel Version 12.5.0: Sun Sep 29 13:33:47 PDT 2013; root:xnu-2050.48.12~1/RELEASE_X86_64 x86_64 BOOST_LIB_VERSION=1_49 +2015-03-10T14:21:24.485+1100 I CONTROL [initandlisten] allocator: system +2015-03-10T14:21:24.486+1100 I CONTROL [initandlisten] options: { net: { port: 27017 }, processManagement: { fork: true }, storage: { dbPath: "/Users/tr/Documents/tmp/data/db", engine: "wiredTiger" }, systemLog: { destination: "file", logAppend: true, path: "/Users/tr/Documents/tmp/data/mongod.log" } } +2015-03-10T14:21:24.487+1100 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0), +2015-03-10T14:21:24.699+1100 I NETWORK [initandlisten] waiting for connections on port 27017 +2015-03-10T14:21:24.739+1100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:63449 #1 (1 connection now open) +2015-03-10T14:21:24.740+1100 I NETWORK [conn1] end connection 127.0.0.1:63449 (0 connections now open) +2015-03-10T14:21:24.741+1100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:63450 #2 (1 connection now open) +2015-03-10T14:21:24.742+1100 I NETWORK [conn2] end connection 127.0.0.1:63450 (0 connections now open) +2015-03-10T14:21:29.843+1100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:63451 #3 (1 connection now open) +2015-03-10T14:21:29.844+1100 I NETWORK [conn3] end connection 127.0.0.1:63451 (0 connections now open) +2015-03-10T14:21:29.845+1100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:63452 #4 (1 connection now open) +2015-03-10T14:21:29.846+1100 I COMMAND [conn4] terminating, shutdown command received +2015-03-10T14:21:29.846+1100 I CONTROL [conn4] now exiting +2015-03-10T14:21:29.846+1100 I NETWORK [conn4] shutdown: going to close listening sockets... +2015-03-10T14:21:29.846+1100 I NETWORK [conn4] closing listening socket: 7 +2015-03-10T14:21:29.846+1100 I NETWORK [conn4] closing listening socket: 8 +2015-03-10T14:21:29.847+1100 I NETWORK [conn4] removing socket file: /tmp/mongodb-27017.sock +2015-03-10T14:21:29.847+1100 I NETWORK [conn4] shutdown: going to flush diaglog... +2015-03-10T14:21:29.847+1100 I NETWORK [conn4] shutdown: going to close sockets... +2015-03-10T14:21:29.847+1100 I STORAGE [conn4] WiredTigerKVEngine shutting down +2015-03-10T14:21:30.202+1100 I STORAGE [conn4] shutdown: removing fs lock... +2015-03-10T14:21:30.202+1100 I CONTROL [conn4] dbexit: rc: 0 diff --git a/mtools/test/test_util_logfile.py b/mtools/test/test_util_logfile.py index 58b26e81..9d310700 100644 --- a/mtools/test/test_util_logfile.py +++ b/mtools/test/test_util_logfile.py @@ -58,6 +58,23 @@ def test_rollover_detection(self): assert logfile.year_rollover == logfile.end + def test_storage_engine_detection(self): + """ LogFile: test if the correct storage engine is detected """ + + logfile = LogFile(self.file_year_rollover) + assert logfile.storage_engine == None + + logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log') + mmapv1 = open(logfile_path, 'r') + logfile = LogFile(mmapv1) + assert logfile.storage_engine == 'mmapv1' + + logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'wiredtiger.log') + wiredtiger = open(logfile_path, 'r') + logfile = LogFile(wiredtiger) + assert logfile.storage_engine == 'wiredTiger' + + def test_hostname_port(self): # mongod logfile_path = os.path.join(os.path.dirname(mtools.__file__), 'test/logfiles/', 'mongod_26.log') diff --git a/mtools/util/logfile.py b/mtools/util/logfile.py index 43fe24d0..1720c763 100644 --- a/mtools/util/logfile.py +++ b/mtools/util/logfile.py @@ -31,6 +31,8 @@ def __init__(self, filehandle): self._repl_set_members = None self._repl_set_version = None + self._storage_engine = None + self._datetime_format = None self._year_rollover = None @@ -145,25 +147,33 @@ def versions(self): @property def repl_set(self): - """ return the replSet(if available). """ + """ return the replSet (if available). """ if not self._num_lines: self._iterate_lines() return self._repl_set @property def repl_set_members(self): - """ return the replSet(if available). """ + """ return the replSet (if available). """ if not self._num_lines: self._iterate_lines() return self._repl_set_members @property def repl_set_version(self): - """ return the replSet(if available). """ + """ return the replSet (if available). """ if not self._num_lines: self._iterate_lines() return self._repl_set_version + @property + def storage_engine(self): + """ return storage engine if available """ + if not self._num_lines: + self._iterate_lines() + return self._storage_engine + + def next(self): """ get next line, adjust for year rollover and hint datetime format. """ @@ -254,6 +264,12 @@ def _iterate_lines(self): if match: self._repl_set = match.group('replSet') + match = re.search('engine: "(?P\S+)"', line) + if match: + self._storage_engine = match.group('engine') + else: + self._storage_engine = 'mmapv1' + if "command admin.$cmd command: { replSetInitiate:" in line: match = re.search('{ _id: "(?P\S+)", members: (?P[^]]+ ])', line) if match: From bdc14cffa26e8cce8fac07dcd70cdc3df22567f6 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 15:38:21 +1100 Subject: [PATCH 18/22] set x-axis bounds correctly on multiple files. Fixes #322. --- mtools/mplotqueries/mplotqueries.py | 4 +++- mtools/mplotqueries/plottypes/base_type.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mtools/mplotqueries/mplotqueries.py b/mtools/mplotqueries/mplotqueries.py index fe7d6b8c..0cd48d02 100644 --- a/mtools/mplotqueries/mplotqueries.py +++ b/mtools/mplotqueries/mplotqueries.py @@ -172,7 +172,9 @@ def parse_logevents(self): # store start and end for each logfile (also works for system.profile and stdin stream) - self.plot_instance.date_range = (logfile.start, logfile.end) + range_min = min(self.plot_instance.date_range[0], logfile.start) + range_max = max(self.plot_instance.date_range[1], logfile.end) + self.plot_instance.date_range = (range_min, range_max) # clear progress bar if self.logfiles and self.progress_bar_enabled: diff --git a/mtools/mplotqueries/plottypes/base_type.py b/mtools/mplotqueries/plottypes/base_type.py index 863d3c83..3dacee3e 100644 --- a/mtools/mplotqueries/plottypes/base_type.py +++ b/mtools/mplotqueries/plottypes/base_type.py @@ -4,6 +4,7 @@ import re from datetime import MINYEAR, MAXYEAR, datetime, timedelta +from dateutil.tz import tzutc import types try: @@ -24,8 +25,7 @@ class BasePlotType(object): sort_order = 0 plot_type_str = 'base' default_group_by = None - date_range = (datetime(MAXYEAR, 12, 31), datetime(MINYEAR, 1, 1)) - + date_range = (datetime(MAXYEAR, 12, 31, tzinfo=tzutc()), datetime(MINYEAR, 1, 1, tzinfo=tzutc())) def __init__(self, args=None, unknown_args=None): self.args = args From 873a62fb923244558c7d57659c2e5909275f899b Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 15:49:30 +1100 Subject: [PATCH 19/22] changed "plotted" to "printed" --- mtools/mlogvis/mlogvis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtools/mlogvis/mlogvis.py b/mtools/mlogvis/mlogvis.py index 219ddb48..bb39d75b 100755 --- a/mtools/mlogvis/mlogvis.py +++ b/mtools/mlogvis/mlogvis.py @@ -18,7 +18,7 @@ def __init__(self): a browser. Automatically opens a browser tab and shows the file.' self.argparser.add_argument('--no-browser', action='store_true', help='only creates .html file, but does not open the browser.') self.argparser.add_argument('--out', '-o', action='store', default=None, help='filename to output. Default is .html') - self.argparser.add_argument('--line-max', action='store', default='10000', help='max count of datapoints at which actual log line strings are not plotted any more.') + self.argparser.add_argument('--line-max', action='store', default='10000', help='max count of datapoints at which actual log line strings are not printed any more.') def _export(self, with_line_str=True): From 460395fb71ecaa68004b9d35d3f1f99141601d9f Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 15:54:59 +1100 Subject: [PATCH 20/22] fixed test for mloginfo. --- mtools/test/test_mloginfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mtools/test/test_mloginfo.py b/mtools/test/test_mloginfo.py index b2daf821..cb7b559e 100644 --- a/mtools/test/test_mloginfo.py +++ b/mtools/test/test_mloginfo.py @@ -67,12 +67,13 @@ def test_28(self): assert results['length'] == '25' assert results['binary'] == 'mongod' assert results['version'] == '2.7.8' + assert results['storage'] == 'mmapv1' def test_28_no_restart(self): self._test_init('mongod_278_partial.log') self.tool.run('%s' % self.logfile_path) lines = sys.stdout.getvalue().splitlines() - assert any(map(lambda line: '>= 2.8 (iso8601 format, level, component)' in line, lines)) + assert any(map(lambda line: '>= 3.0 (iso8601 format, level, component)' in line, lines)) def test_multiple_files(self): self.tool.run('%s %s' % (self.logfile_path, self.logfile_path)) From bdf68bf51b446bf9d665c866c9029f9697ca6bc3 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 16:04:25 +1100 Subject: [PATCH 21/22] updated README and CHANGES. bumped version to 1.1.8. --- CHANGES.md | 11 +++++++++++ README.md | 2 +- mtools/version.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8adb6f9e..497bd64c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,17 @@ Changes to mtools ================= #### version 1.1.8 + + * mloginfo: storage engine is now listed for log files (#330) + * mplotqueries: x-axis bounds corrected when parsing multiple files (#322) + * mlogfilter: truncated log lines ("too long ...") recognized and parsed as much as possible (#133) + * better cross-platform script support, especially for window susers (#230) + * logging components are updated to match final version of MongoDB 3.0 (#328, #327) + * removed hard dependency on pymongo, only required if mlaunch is used (#337) + * removed deprecated scripts like mlogversion, mlogdistinct (#336) + * command in LogEvent is now always lowercase (#335) + * LogEvent now has writeConflicts property (#334) + * documented matplotlib minimum version 1.8.0 (#332) #### version 1.1.7 diff --git a/README.md b/README.md index 84a570e5..2a66736b 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ instructions for these modules. Recent Changes -------------- -The current version of mtools is 1.1.8-dev. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools. +The current version of mtools is 1.1.8. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools. Contribute to mtools diff --git a/mtools/version.py b/mtools/version.py index 003f37d1..f0be1393 100644 --- a/mtools/version.py +++ b/mtools/version.py @@ -1 +1 @@ -__version__ = '1.1.8-dev' +__version__ = '1.1.8' From 3d6b22849d1ff230f091b37fe088a8c0505d8268 Mon Sep 17 00:00:00 2001 From: Thomas Rueckstiess Date: Tue, 10 Mar 2015 16:27:39 +1100 Subject: [PATCH 22/22] some more pymongo dependencies that were unchecked. --- mtools/mloginfo/sections/connection_section.py | 7 +++++-- mtools/mloginfo/sections/distinct_section.py | 6 +++++- mtools/mloginfo/sections/query_section.py | 1 - mtools/mloginfo/sections/restart_section.py | 5 ++++- mtools/util/cmdlinetool.py | 10 ++++++++-- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/mtools/mloginfo/sections/connection_section.py b/mtools/mloginfo/sections/connection_section.py index 24ae7c8b..30982ffc 100644 --- a/mtools/mloginfo/sections/connection_section.py +++ b/mtools/mloginfo/sections/connection_section.py @@ -2,8 +2,11 @@ from collections import defaultdict import re -from mtools.util.profile_collection import ProfileCollection - +try: + from mtools.util.profile_collection import ProfileCollection +except ImportError: + ProfileCollection = None + class ConnectionSection(BaseSection): """ This section goes through the logfile and extracts information about opened and closed connections. diff --git a/mtools/mloginfo/sections/distinct_section.py b/mtools/mloginfo/sections/distinct_section.py index 60bac0ea..a6fd75f6 100644 --- a/mtools/mloginfo/sections/distinct_section.py +++ b/mtools/mloginfo/sections/distinct_section.py @@ -1,7 +1,11 @@ from base_section import BaseSection from mtools.util.log2code import Log2CodeConverter -from mtools.util.profile_collection import ProfileCollection + +try: + from mtools.util.profile_collection import ProfileCollection +except ImportError: + ProfileCollection = None from collections import defaultdict diff --git a/mtools/mloginfo/sections/query_section.py b/mtools/mloginfo/sections/query_section.py index 107556ae..a57cc951 100644 --- a/mtools/mloginfo/sections/query_section.py +++ b/mtools/mloginfo/sections/query_section.py @@ -1,6 +1,5 @@ from base_section import BaseSection -from mtools.util.profile_collection import ProfileCollection from mtools.util.grouping import Grouping from mtools.util.print_table import print_table from mtools.util import OrderedDict diff --git a/mtools/mloginfo/sections/restart_section.py b/mtools/mloginfo/sections/restart_section.py index 82a8af48..64af2e4b 100644 --- a/mtools/mloginfo/sections/restart_section.py +++ b/mtools/mloginfo/sections/restart_section.py @@ -1,6 +1,9 @@ from base_section import BaseSection -from mtools.util.profile_collection import ProfileCollection +try: + from mtools.util.profile_collection import ProfileCollection +except ImportError: + ProfileCollection = None class RestartSection(BaseSection): """ This section determines if there were any restarts in the log file and prints out diff --git a/mtools/util/cmdlinetool.py b/mtools/util/cmdlinetool.py index 6d9d0fdc..02cc7395 100644 --- a/mtools/util/cmdlinetool.py +++ b/mtools/util/cmdlinetool.py @@ -49,13 +49,19 @@ def __call__(self, string): if host == 'localhost' or re.match('\d+\.\d+\.\d+\.\d+', host): return ProfileCollection(host, port, database, collection) - raise argparse.ArgumentTypeError("can't parse %s as file or MongoDB connection string." % string) + raise argparse.ArgumentTypeError("can't open %s as file or MongoDB connection string." % string) except ImportError: class InputSourceAction(argparse.FileType): - pass + def __call__(self, string): + try: + # catch filetype and return LogFile object + filehandle = argparse.FileType.__call__(self, string) + return LogFile(filehandle) + except argparse.ArgumentTypeError as e: + raise argparse.ArgumentTypeError("can't open %s" % string) class BaseCmdLineTool(object):