[MXNET-545] Fix broken cython build

Jenkinsfile

@@ -22,8 +22,10 @@
 
 // mxnet libraries
 mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_cython_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 // for scala build, need to pass extra libs when run with dist_kvstore
 mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_cython_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, python/mxnet/_cy2/*.so, python/mxnet/_cy3/*.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
@@ -76,7 +78,7 @@ def init_git_win() {
 def pack_lib(name, libs=mx_lib) {
   sh """
 echo "Packing ${libs} into ${name}"
-echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done
 """
   stash includes: libs, name: name
 }
@@ -86,7 +88,7 @@ def unpack_lib(name, libs=mx_lib) {
   unstash name
   sh """
 echo "Unpacked ${libs} from ${name}"
-echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+for i in \$(echo ${libs} | sed -e 's/,/ /g'); do md5sum \$i; done
 """
 }
 
@@ -139,6 +141,12 @@ def python3_ut(docker_container_name) {
   }
 }
 
+def python3_cython_ut(docker_container_name) {
+  timeout(time: max_time, unit: 'MINUTES') {
+    docker_run(docker_container_name, 'unittest_ubuntu_python3_cython_cpu', false)
+  }
+}
+
 def python3_ut_mkldnn(docker_container_name) {
   timeout(time: max_time, unit: 'MINUTES') {
     docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false)
@@ -154,6 +162,12 @@ def python2_gpu_ut(docker_container_name) {
   }
 }
 
+def python2_cython_gpu_ut(docker_container_name) {
+  timeout(time: max_time, unit: 'MINUTES') {
+    docker_run(docker_container_name, 'unittest_ubuntu_python2_cython_gpu', true)
+  }
+}
+
 // Python 3
 def python3_gpu_ut(docker_container_name) {
   timeout(time: max_time, unit: 'MINUTES') {
@@ -228,7 +242,7 @@ try {
           timeout(time: max_time, unit: 'MINUTES') {
             init_git()
             docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
-            pack_lib('cpu', mx_dist_lib)
+            pack_lib('cpu', mx_cython_dist_lib)
           }
         }
       }
@@ -314,7 +328,7 @@ try {
           timeout(time: max_time, unit: 'MINUTES') {
             init_git()
             docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false)
-            pack_lib('gpu', mx_dist_lib)
+            pack_lib('gpu', mx_cython_dist_lib)
             stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet'
             stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet'
             stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet'
@@ -564,8 +578,8 @@ try {
         ws('workspace/ut-python3-cpu') {
           try {
             init_git()
-            unpack_lib('cpu')
-            python3_ut('ubuntu_cpu')
+            unpack_lib('cpu', mx_cython_lib)
+            python3_cython_ut('ubuntu_cpu')
             publish_test_coverage()
           } finally {
             collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml')
@@ -579,8 +593,8 @@ try {
         ws('workspace/ut-python2-gpu') {
           try {
             init_git()
-            unpack_lib('gpu', mx_lib)
-            python2_gpu_ut('ubuntu_gpu')
+            unpack_lib('gpu', mx_cython_lib)
+            python2_cython_gpu_ut('ubuntu_gpu')
             publish_test_coverage()
           } finally {
             collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml')

Makefile

@@ -541,7 +541,7 @@ doxygen:
 
 # Cython build
 cython:
-	cd python; python setup.py build_ext --inplace --with-cython
+	cd python; $(PYTHON) setup.py build_ext --inplace --with-cython
 
 cython2:
 	cd python; python2 setup.py build_ext --inplace --with-cython

ci/docker/install/ubuntu_python.sh

@@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py
 python3 get-pip.py
 python2 get-pip.py
 
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 Cython==0.28.2
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 Cython==0.28.2

ci/docker/runtime_functions.sh

@@ -33,6 +33,18 @@ clean_repo() {
     git submodule update --init --recursive
 }
 
+check_cython() {
+    set -ex
+    local python_ver=$1
+    if [ "$(echo -e 'import mxnet as mx\nprint(mx.nd._internal.NDArrayBase.__module__)' | python${python_ver})" != "mxnet._cy${python_ver}.ndarray" ]; then
+        echo "ERROR: cython is not used."
+        return 1
+    else
+        echo "NOTE: cython is used."
+        return 0
+    fi
+}
+
 build_ccache_wrappers() {
     set -ex
 
@@ -343,6 +355,11 @@ build_ubuntu_cpu_openblas() {
         -j$(nproc)
 
     report_ccache_usage
+
+    export CC="gcc"
+    export CXX="g++"
+    make cython PYTHON=python2
+    make cython PYTHON=python3
 }
 
 build_ubuntu_cpu_clang39() {
@@ -484,6 +501,9 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_CPP_PACKAGE=1             \
         USE_DIST_KVSTORE=1            \
         -j$(nproc)
+
+    make cython PYTHON=python2
+    make cython PYTHON=python3
 }
 
 build_ubuntu_amalgamation() {
@@ -575,6 +595,19 @@ unittest_ubuntu_python3_cpu() {
     nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
 }
 
+unittest_ubuntu_python3_cython_cpu() {
+    set -ex
+    export PYTHONPATH=./python/
+    # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+    # /~https://github.com/apache/incubator-mxnet/issues/10026
+    #export MXNET_MKLDNN_DEBUG=1  # Ignored if not present
+    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+    export MXNET_ENFORCE_CYTHON=1
+    check_cython 3
+    nosetests-3.4 --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
+    nosetests-3.4 --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
+}
+
 unittest_ubuntu_python3_cpu_mkldnn() {
     set -ex
     export PYTHONPATH=./python/
@@ -596,6 +629,18 @@ unittest_ubuntu_python2_gpu() {
     nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
 }
 
+unittest_ubuntu_python2_cython_gpu() {
+    set -ex
+    export PYTHONPATH=./python/
+    # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+    # /~https://github.com/apache/incubator-mxnet/issues/10026
+    #export MXNET_MKLDNN_DEBUG=1  # Ignored if not present
+    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+    export MXNET_ENFORCE_CYTHON=1
+    check_cython 2
+    nosetests-2.7 --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
+}
+
 tutorialtest_ubuntu_python3_gpu() {
     set -ex
     cd /work/mxnet/docs

docs/faq/env_var.md

@@ -137,6 +137,19 @@ When USE_PROFILER is enabled in Makefile or CMake, the following environments ca
 	- If set to '0', profiler records the events of the symbolic operators.
 	- If set to '1', profiler records the events of all operators.
 
+## Interface between Python and the C API
+
+* MXNET_ENABLE_CYTHON
+  - Values: 0(false), 1(true) ```(default=1)```
+  - If set to 0, MXNet uses the ctypes to interface with the C API.
+  - If set to 1, MXNet tries to use the cython modules for the ndarray and symbol. If it fails, the ctypes is used or an error occurs depending on MXNET_ENFORCE_CYTHON.
+
+* MXNET_ENFORCE_CYTHON
+  - Values: 0(false) or 1(true) ```(default=0)```
+  - This has an effect only if MXNET_ENABLE_CYTHON is 1.
+  - If set to 0, MXNet fallbacks to the ctypes if importing the cython modules fails.
+  - If set to 1, MXNet raises an error if importing the cython modules fails.
+
 ## Other Environment Variables
 
 * MXNET_CUDNN_AUTOTUNE_DEFAULT
@@ -146,7 +159,7 @@ When USE_PROFILER is enabled in Makefile or CMake, the following environments ca
   - Performance tests are run to pick the convolution algo when value is 1 or 2
   - Value of 1 chooses the best algo in a limited workspace
   - Value of 2 chooses the fastest algo whose memory requirements may be larger than the default workspace threshold
-  
+
 
 * MXNET_GLUON_REPO
   - Values: String ```(default='https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'```

docs/install/index.md

@@ -459,7 +459,16 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas
 $ sudo apt-get install -y python-dev python-setuptools python-pip libgfortran3
 ```
 
-**Step 2** Install the MXNet Python binding.
+**Step 2** Build cython modules (optional).
+
+```bash
+$ sudo apt-get install -y cython
+$ make cython # You can set the python executable with `PYTHON` flag, e.g., make cython PYTHON=python3
+```
+
+Note that you can control the use of the cython modules at runtime via the environment variables `MXNET_ENABLE_CYTHON` and `MXNET_ENFORCE_CYTHON`. See [here](/faq/env_var.html) for details.
+
+**Step 3** Install the MXNet Python binding.
 
 ```bash
 $ cd python
@@ -468,13 +477,13 @@ $ pip install -e .
 
 Note that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed.
 
-**Step 3**  Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package).
+**Step 4**  Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package).
 ```bash
 sudo apt-get install graphviz
 pip install graphviz
 ```
 
-**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation).
+**Step 5** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation).
 
 </div><!-- END of build from source -->
 </div><!-- END of CPU -->

make/config.mk

@@ -196,6 +196,9 @@ EXTRA_OPERATORS =
 # Create C++ interface package
 USE_CPP_PACKAGE = 0
 
+# Python executable. Needed for cython target
+PYTHON = python
+
 #----------------------------
 # plugins
 #----------------------------

python/mxnet/cython/base.pyi

@@ -90,20 +90,25 @@ cdef extern from "mxnet/c_api.h":
     int MXSymbolSetAttr(SymbolHandle symbol,
                         const char* key,
                         const char* value);
-    int MXImperativeInvoke(OpHandle creator,
+    int MXImperativeInvokeEx(OpHandle creator,
+                             int num_inputs,
+                             NDArrayHandle *inputs,
+                             int *num_outputs,
+                             NDArrayHandle **outputs,
+                             int num_params,
+                             const char **param_keys,
+                             const char **param_vals,
+                             const int **out_stypes);
+    int MXNDArrayFree(NDArrayHandle handle);
+    int MXCreateCachedOpEx(SymbolHandle handle,
+                            int num_flags,
+                            const char** keys,
+                            const char** vals,
+                            CachedOpHandle *out);
+    int MXFreeCachedOp(CachedOpHandle handle);
+    int MXInvokeCachedOpEx(CachedOpHandle handle,
                            int num_inputs,
                            NDArrayHandle *inputs,
                            int *num_outputs,
                            NDArrayHandle **outputs,
-                           int num_params,
-                           const char **param_keys,
-                           const char **param_vals);
-    int MXNDArrayFree(NDArrayHandle handle);
-    int MXCreateCachedOp(SymbolHandle handle,
-                         CachedOpHandle *out);
-    int MXFreeCachedOp(CachedOpHandle handle);
-    int MXInvokeCachedOp(CachedOpHandle handle,
-                       int num_inputs,
-                       NDArrayHandle *inputs,
-                       int *num_outputs,
-                       NDArrayHandle **outputs);
+                           const int **out_stypes);