flavoured windows builds

ci/build_windows.py

@@ -20,7 +20,6 @@
 
 """User friendly / multi platform builder script"""
 
-import contextlib
 import subprocess
 import logging
 import os
@@ -33,15 +32,87 @@
 import argparse
 from util import *
 import json
+from enum import Enum
+import time
+import datetime
+import shutil
+import glob
+from distutils.dir_util import copy_tree
+
+class BuildFlavour(Enum):
+    WIN_CPU = 'WIN_CPU'
+    WIN_CPU_MKLDNN = 'WIN_CPU_MKLDNN'
+    WIN_GPU = 'WIN_GPU'
+    WIN_GPU_MKLDNN = 'WIN_GPU_MKLDNN'
+
+CMAKE_FLAGS = {
+#    'CPU': '-DUSE_CUDA=0 \
+#            -DUSE_CUDNN=0 \
+#            -DUSE_NVRTC=0 \
+#            -DUSE_OPENCV=1 \
+#            -DUSE_OPENMP=1 \
+#            -DUSE_PROFILER=1 \
+#            -DUSE_BLAS=open \
+#            -DUSE_LAPACK=1 \
+#            -DBUILD_CPP_EXAMPLES=1 \
+#            -DUSE_DIST_KVSTORE=0'
+    'WIN_CPU': '-DUSE_CUDA=0 \
+                -DUSE_CUDNN=0 \
+                -DUSE_NVRTC=0 \
+                -DUSE_OPENCV=1 \
+                -DUSE_OPENMP=1 \
+                -DUSE_PROFILER=1 \
+                -DUSE_BLAS=open \
+                -DUSE_LAPACK=1 \
+                -DUSE_DIST_KVSTORE=0 \
+                -DUSE_MKL_IF_AVAILABLE=0'
+
+    ,'WIN_CPU_MKLDNN': '-DUSE_CUDA=0 \
+                        -DUSE_CUDNN=0 \
+                        -DUSE_NVRTC=0 \
+                        -DUSE_OPENCV=1 \
+                        -DUSE_OPENMP=1 \
+                        -DUSE_PROFILER=1 \
+                        -DUSE_BLAS=open \
+                        -DUSE_LAPACK=1 \
+                        -DUSE_DIST_KVSTORE=0 \
+                        -DUSE_MKL_IF_AVAILABLE=1'
+
+    ,'WIN_GPU': '-DUSE_CUDA=1 \
+                 -DUSE_CUDNN=1 \
+                 -DUSE_NVRTC=1 \
+                 -DUSE_OPENCV=1  \
+                 -DUSE_OPENMP=1 \
+                 -DUSE_PROFILER=1 \
+                 -DUSE_BLAS=open  \
+                 -DUSE_LAPACK=1  \
+                 -DUSE_DIST_KVSTORE=0 \
+                 -DCUDA_ARCH_NAME=Manual \
+                 -DCUDA_ARCH_BIN=52 \
+                 -DCUDA_ARCH_PTX=52 \
+                 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" \
+                 -DUSE_MKL_IF_AVAILABLE=0 \
+                 -DCMAKE_BUILD_TYPE=Release'
+
+    ,'WIN_GPU_MKLDNN': '-DUSE_CUDA=1 \
+                        -DUSE_CUDNN=1 \
+                        -DUSE_NVRTC=1 \
+                        -DUSE_OPENCV=1 \
+                        -DUSE_OPENMP=1 \
+                        -DUSE_PROFILER=1 \
+                        -DUSE_BLAS=open \
+                        -DUSE_LAPACK=1 \
+                        -DUSE_DIST_KVSTORE=0 \
+                        -DCUDA_ARCH_NAME=Manual \
+                        -DCUDA_ARCH_BIN=52 \
+                        -DCUDA_ARCH_PTX=52 \
+                        -DUSE_MKLDNN=1 \
+                        -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 \
+                        /DNDEBUG" \
+                        -DCMAKE_BUILD_TYPE=Release'
+
+}
 
-@contextlib.contextmanager
-def remember_cwd():
-    '''
-    Restore current directory when exiting context
-    '''
-    curdir = os.getcwd()
-    try: yield
-    finally: os.chdir(curdir)
 
 def get_vsvars_environment(architecture, vsvars):
     """
@@ -62,49 +133,58 @@ def get_vsvars_environment(architecture, vsvars):
 
 
 def windows_build(args):
-    BUILD_BAT=r'''call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat"
-msbuild mxnet.sln /p:configuration=release;platform=x64
-'''
     vsvars_env = get_vsvars_environment(args.arch, args.vsvars)
-    logging.info("vsvars environment: %s", vsvars_env)
+    logging.debug("vsvars environment: %s", vsvars_env)
     os.environ.update(vsvars_env)
 
-    os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas"
-    os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14"
-    os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0"
-    print(os.environ)
-
     path = args.output
-    xmkdir(path)
+    os.makedirs(path, exist_ok=True)
     mxnet_root = get_mxnet_root()
     logging.info("Found mxnet root: {}".format(mxnet_root))
     with remember_cwd():
         os.chdir(path)
         logging.info("Generating project with CMake")
-        check_call("cmake -G \"Visual Studio 14 2015 Win64\" \
-            -DUSE_CUDA=0 \
-            -DUSE_CUDNN=0 \
-            -DUSE_NVRTC=0 \
-            -DUSE_OPENCV=1 \
-            -DUSE_OPENMP=1 \
-            -DUSE_PROFILER=1 \
-            -DUSE_BLAS=open \
-            -DUSE_LAPACK=1 \
-            -DBUILD_CPP_EXAMPLES=1 \
-            -DUSE_DIST_KVSTORE=0 \
-            {}".format(mxnet_root), shell=True)
-
+        check_call("cmake -G \"Visual Studio 14 2015 Win64\" {} {}".format(CMAKE_FLAGS[args.flavour], mxnet_root), shell=True)
+        logging.info("Building with visual studio")
+        t0 = int(time.time())
         check_call(["msbuild", "mxnet.sln","/p:configuration=release;platform=x64"])
-    windows_package(path)
-
-def windows_package():
-    pass
-
+        logging.info("Build flavour: %s complete in directory: \"%s\"", args.flavour, os.path.abspath(path))
+        logging.info("Build took %s" , datetime.timedelta(seconds=int(time.time()-t0)))
+    windows_package(args)
+
+def windows_package(args):
+    pkgfile = 'windows_package.7z'
+    pkgdir = os.path.abspath('windows_package')
+    logging.info("Packaging libraries and headers in package: %s", pkgfile)
+    j = os.path.join
+    pkgdir_lib = os.path.abspath(j(pkgdir, 'lib'))
+    with remember_cwd():
+        os.chdir(args.output)
+        logging.info("Looking for static libraries and dlls in: \"%s", os.getcwd())
+        libs = list(glob.iglob('**/*.lib', recursive=True))
+        dlls = list(glob.iglob('**/*.dll', recursive=True))
+        os.makedirs(pkgdir_lib, exist_ok=True)
+        for lib in libs:
+            logging.info("packing lib: %s", lib)
+            shutil.copy(lib, pkgdir_lib)
+        for dll in dlls:
+            logging.info("packing dll: %s", dll)
+            shutil.copy(lib, pkgdir_lib)
+        os.chdir(get_mxnet_root())
+        logging.info('packing python bindings')
+        copy_tree('python', j(pkgdir, 'python'))
+        logging.info('packing headers')
+        copy_tree('include', j(pkgdir, 'include'))
+        copy_tree(j('3rdparty','dmlc-core','include'), j(pkgdir, 'include'))
+        copy_tree(j('3rdparty','mshadow', 'mshadow'), j(pkgdir, 'include', 'mshadow'))
+        copy_tree(j('3rdparty','tvm','nnvm', 'include'), j(pkgdir,'include', 'nnvm', 'include'))
+        logging.info("Compressing package: %s", pkgfile)
+        check_call(['7z', 'a', pkgfile, pkgdir])
 
 
 def nix_build(args):
     path = args.output
-    xmkdir(path)
+    os.makedirs(path, exist_ok=True)
     with remember_cwd():
         os.chdir(path)
         logging.info("Generating project with CMake")
@@ -119,6 +199,7 @@ def nix_build(args):
 def main():
     logging.getLogger().setLevel(logging.INFO)
     logging.basicConfig(format='%(asctime)-15s %(message)s')
+    logging.info("MXNet Windows build helper")
 
     parser = argparse.ArgumentParser()
     parser.add_argument("-o", "--output",
@@ -136,10 +217,21 @@ def main():
         default='x64',
         type=str)
 
+    parser.add_argument("-f", "--flavour",
+        help="build flavour",
+        default='WIN_CPU',
+        choices=[x.name for x in BuildFlavour],
+        type=str)
+
     args = parser.parse_args()
+    logging.info("Build flavour: %s", args.flavour)
 
     system = platform.system()
     if system == 'Windows':
+        logging.info("Detected Windows platform")
+        os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas"
+        os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14"
+        os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0"
         windows_build(args)
 
     elif system == 'Linux' or system == 'Darwin':

ci/util.py

@@ -1,4 +1,5 @@
 import os
+import contextlib
 
 def get_mxnet_root() -> str:
     curpath = os.path.abspath(os.path.dirname(__file__))
@@ -13,19 +14,13 @@ def is_mxnet_root(path: str) -> bool:
         curpath = parent
     return curpath
 
-def xmkdir(d):
-    rev_path_list = list()
-    head = d
-    while len(head) and head != os.sep:
-        rev_path_list.append(head)
-        (head, tail) = os.path.split(head)
-
-    rev_path_list.reverse()
-    for p in rev_path_list:
-        try:
-            os.mkdir(p)
-        except OSError as e:
-            if e.errno != 17:
-                raise
+@contextlib.contextmanager
+def remember_cwd():
+    '''
+    Restore current directory when exiting context
+    '''
+    curdir = os.getcwd()
+    try: yield
+    finally: os.chdir(curdir)