[rabit harden] fix model recovery tests failures (#510)

* [rabit harden] fix rabit model recovery unit test failure

* fix cpplint isssue in python3

* per feedback cleanup

* make travis osx CXX setting consistent with cmakeetst

* Revert "make travis osx CXX setting consistent with cmakeetst"

This reverts commit 4c84baf.

include/dmlc/concurrency.h

@@ -13,6 +13,7 @@
 #include <queue>
 #include <mutex>
 #include <vector>
+#include <utility>
 #include <condition_variable>
 #include "dmlc/base.h"
 

include/dmlc/memory.h

@@ -7,6 +7,8 @@
 #define DMLC_MEMORY_H_
 
 #include <vector>
+#include <memory>
+#include <utility>
 #include "./base.h"
 #include "./logging.h"
 #include "./thread_local.h"

include/dmlc/optional.h

@@ -25,7 +25,7 @@ struct nullopt_t {
   explicit nullopt_t(int a) {}
 #else
   /*! \brief dummy constructor */
-  constexpr nullopt_t(int a) {}
+  constexpr explicit nullopt_t(int a) {}
 #endif
 };
 

include/dmlc/thread_group.h

@@ -11,6 +11,8 @@
 #include <dmlc/logging.h>
 #include <string>
 #include <mutex>
+#include <utility>
+#include <memory>
 #include <set>
 #include <thread>
 #include <unordered_set>

tracker/dmlc_tracker/local.py

@@ -13,28 +13,42 @@ def exec_cmd(cmd, role, taskid, pass_env):
     """Execute the command line command."""
     if cmd[0].find('/') == -1 and os.path.exists(cmd[0]) and os.name != 'nt':
         cmd[0] = './' + cmd[0]
-    cmd = ' '.join(cmd)
+    cmdline = ' '.join(cmd)
     env = os.environ.copy()
     for k, v in pass_env.items():
         env[k] = str(v)
 
     env['DMLC_TASK_ID'] = str(taskid)
     env['DMLC_ROLE'] = role
     env['DMLC_JOB_CLUSTER'] = 'local'
-
     num_retry = env.get('DMLC_NUM_ATTEMPT', 0)
 
+    #overwrite default num of retry with commandline value
+    for param in cmd:
+        if param.startswith('DMLC_NUM_ATTEMPT'):
+            num_retry = int(param.split('=')[1])
+    logging.debug('num of retry %d',num_retry)
+
     while True:
         if os.name == 'nt':
-            ret = subprocess.call(cmd, shell=True, env=env)
+            ret = subprocess.call(cmdline, shell=True, env=env)
         else:
-            ret = subprocess.call(cmd, shell=True, executable='bash', env=env)
+            ret = subprocess.call(cmdline, shell=True, executable='bash', env=env)
         if ret == 0:
             logging.debug('Thread %d exit with 0', taskid)
             return
         else:
             num_retry -= 1
+            newcmd = []
             if num_retry >= 0:
+                # failure trail increase by 1 and restart failed worker
+                for arg in cmd:
+                    if arg.startswith('rabit_num_trial'):
+                        val = arg.split('=')[1]
+                        arg = arg.replace(val, str(int(val)+1))
+                    newcmd.append(arg)
+                cmdline = ' '.join(newcmd)
+                cmd = newcmd
                 continue
             if os.name == 'nt':
                 sys.exit(-1)