From 8b99ff37b15af9f3d72be0b6aae6a4765c775908 Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Mon, 3 Dec 2018 17:27:37 -0800 Subject: [PATCH] [WIP]: improve scala maven jni build and packing. --- .../assembly/linux-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ .../assembly/linux-x86_64-gpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ .../osx-x86_64-cpu/main/assembly/assembly.xml | 30 ---------- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 ++ .../src/main/assembly/assembly.xml | 6 ++ scala-package/core/pom.xml | 8 +++ .../mxnet/util/NativeLibraryLoader.scala | 55 ++++++++++++------- scala-package/examples/pom.xml | 4 ++ scala-package/infer/pom.xml | 4 ++ .../init-native/linux-x86_64/pom.xml | 42 +++++++++++--- scala-package/init-native/osx-x86_64/pom.xml | 49 +++++++++++++++-- scala-package/native/linux-x86_64-cpu/pom.xml | 25 +++++---- scala-package/native/linux-x86_64-gpu/pom.xml | 25 +++++---- scala-package/native/osx-x86_64-cpu/pom.xml | 50 ++++++++++++++--- scala-package/pom.xml | 2 + 17 files changed, 228 insertions(+), 96 deletions(-) delete mode 100644 scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index fbc0ab027ac7..9ee0e0270b07 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -14,6 +14,10 @@ MXNet Scala Package - Full Linux-x86_64 CPU-only jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml index a574f8af25d9..f4c2017c8241 100644 --- a/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-cpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index a1a94808e918..88478a56b7f4 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -14,6 +14,10 @@ MXNet Scala Package - Full Linux-x86_64 GPU jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml index 3a064bf9f2ce..2aca64bdf1a9 100644 --- a/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/linux-x86_64-gpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml deleted file mode 100644 index fecafecad31e..000000000000 --- a/scala-package/assembly/osx-x86_64-cpu/main/assembly/assembly.xml +++ /dev/null @@ -1,30 +0,0 @@ - - full - - jar - - false - - - - *:*:jar - - / - true - true - runtime - - - lib/native - ${artifact.artifactId}${dashClassifier?}.${artifact.extension} - false - false - false - - *:*:dll:* - *:*:so:* - *:*:jnilib:* - - - - diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index bb6af0353762..b1dd49394a0e 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -14,6 +14,10 @@ MXNet Scala Package - Full OSX-x86_64 CPU-only jar + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet diff --git a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml index bdbd09f170c0..e9bc3728fcd0 100644 --- a/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml +++ b/scala-package/assembly/osx-x86_64-cpu/src/main/assembly/assembly.xml @@ -25,4 +25,10 @@ + + + ${MXNET_DIR}/lib/libmxnet.so + lib/native + + diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 3425bb15f62a..ca2c773e0a92 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -12,6 +12,7 @@ true + ${project.parent.basedir}/.. mxnet-core_2.11 @@ -77,6 +78,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + @@ -88,6 +92,10 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target ${skipTests} + always + + ${MXNET_DIR}/lib + diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala index e94d320391fa..7296e4785a54 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala @@ -85,12 +85,10 @@ private[mxnet] object NativeLibraryLoader { } logger.debug(s"Attempting to load $loadLibname") val libFileInJar = libPathInJar + loadLibname - val is: InputStream = getClass.getResourceAsStream(libFileInJar) - if (is == null) { - throw new UnsatisfiedLinkError(s"Couldn't find the resource $loadLibname") - } - logger.info(s"Loading $loadLibname from $libPathInJar copying to $libname") - loadLibraryFromStream(libname, is) + saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so") + val tempfile: File = saveLibraryToTemp(libname, libFileInJar) + + loadLibraryFromFile(libname, tempfile) } /** @@ -109,7 +107,7 @@ private[mxnet] object NativeLibraryLoader { @throws(classOf[IOException]) private def createTempFile(name: String): File = { - new File(_tempDir + File.separator + name) + new File(_tempDir, name) } /** @@ -117,11 +115,34 @@ private[mxnet] object NativeLibraryLoader { * and loads from there. * * @param libname name of the library (just used in constructing the library name) - * @param is InputStream pointing to the library + * @param tempfile File pointing to the library */ - private def loadLibraryFromStream(libname: String, is: InputStream) { + private def loadLibraryFromFile(libname: String, tempfile: File) { + try { + logger.debug("Loading library from {}", tempfile.getPath) + System.load(tempfile.getPath) + } catch { + case ule: UnsatisfiedLinkError => + logger.error("Couldn't load copied link file: {}", ule.toString) + throw ule + } + } + + /** + * Load a system library from a stream. Copies the library to a temp file + * and loads from there. + * + * @param libname name of the library (just used in constructing the library name) + * @param resource String resource path in the jar file + */ + private def saveLibraryToTemp(libname: String, resource: String): File = { try { - val tempfile: File = createTempFile(libname) + val is: InputStream = getClass.getResourceAsStream(resource) + if (is == null) { + throw new UnsatisfiedLinkError(s"Couldn't find the resource $resource") + } + + val tempfile: File = new File(_tempDir, libname) val os: OutputStream = new FileOutputStream(tempfile) logger.debug("tempfile.getPath() = {}", tempfile.getPath) val savedTime: Long = System.currentTimeMillis @@ -131,20 +152,14 @@ private[mxnet] object NativeLibraryLoader { os.write(buf, 0, len) len = is.read(buf) } - os.flush() - val lock: InputStream = new FileInputStream(tempfile) os.close() + is.close() val seconds: Double = (System.currentTimeMillis - savedTime).toDouble / 1e3 - logger.debug(s"Copying took $seconds seconds.") - logger.debug("Loading library from {}", tempfile.getPath) - System.load(tempfile.getPath) - lock.close() + logger.debug(s"Copying $libname took $seconds seconds.") + tempfile } catch { case io: IOException => - logger.error("Could not create the temp file: {}", io.toString) - case ule: UnsatisfiedLinkError => - logger.error("Couldn't load copied link file: {}", ule.toString) - throw ule + throw new UnsatisfiedLinkError(s"Could not create temp file for $libname") } } } diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 9e8e119c3c4f..08a709c2684c 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -15,6 +15,7 @@ true + ${project.parent.basedir}/.. @@ -137,6 +138,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 3e6980cb6f4b..362aa4e84e88 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -15,6 +15,7 @@ true + ${project.parent.basedir}/.. @@ -77,6 +78,9 @@ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties + + ${MXNET_DIR}/lib + diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index 12a36bd6e944..a4c6033b13d4 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -16,6 +16,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,22 +66,24 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - ${all_ldpaths} -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive + -Wl,--no-whole-archive -pthread -lm -fopenmp -lrt - ${ldflags} - -fopenmp + -Wl,-rpath=${dollor}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +92,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} @@ -101,6 +106,25 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index d0290942ef84..dce8081ef23c 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -16,6 +16,10 @@ jnilib + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,8 +66,14 @@ -std=c++0x - -I${project.basedir}/../../../include - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared @@ -72,11 +82,9 @@ -framework JavaVM -Wl,-exported_symbol,_Java_* -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - ${ldflags} + -lmxnet -L${MXNET_DIR}/lib @@ -85,7 +93,6 @@ javah generate-sources - darwin default ${project.build.directory}/custom-javah ${basedir} @@ -100,6 +107,36 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib + + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index ac8e4a45e67a..cb5c0fefe690 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -16,6 +16,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,22 +66,20 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - ${ldflags} - -fopenmp + -Wl,-rpath=${dollor}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +88,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index cdba5774f6a0..0101ae9e1d0d 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -16,6 +16,10 @@ so + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,22 +66,20 @@ -std=c++0x - -I${project.basedir}/../../../include - ${all_includes} - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -O3 -DNDEBUG=1 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared - - ${all_ldpaths} - -Wl,--whole-archive - ${lddeps} - -Wl,--no-whole-archive - - ${ldflags} - -fopenmp + -Wl,-rpath=${dollor}ORIGIN -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +88,6 @@ javah generate-sources - linux default ${project.build.directory}/custom-javah ${basedir} diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 333486c67392..abb75f6449d9 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -16,6 +16,10 @@ jnilib + + ${project.parent.parent.basedir}/.. + + org.apache.mxnet @@ -62,8 +66,14 @@ -std=c++0x - -I../../../include - ${cflags} + -I${MXNET_DIR}/include + -I${MXNET_DIR}/3rdparty/dmlc-core/include + -I${MXNET_DIR}/3rdparty/mshadow + -I${MXNET_DIR}/3rdparty/dlpack/include + -I${MXNET_DIR}/3rdparty/tvm/nnvm/include + -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=0 + -g -O0 -fPIC -msse3 -mf16c + -Wall -Wsign-compare -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs -shared @@ -72,12 +82,9 @@ -framework JavaVM -Wl,-exported_symbol,_Java_* -Wl,-x - ${lddeps} - -force_load ${project.basedir}/../../../lib/libmxnet.a - -force_load ${project.basedir}/../../../3rdparty/tvm/nnvm/lib/libnnvm.a - ${ldflags} + -Wl,-install_name,libmxnet-scala.jnilib -lmxnet -L${MXNET_DIR}/lib @@ -86,7 +93,6 @@ javah generate-sources - darwin default ${project.build.directory}/custom-javah ${basedir} @@ -101,6 +107,36 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + post-native-build + package + + exec + + + install_name_tool + -change lib/libmxnet.so @loader_path/libmxnet.so ${project.build.directory}/${artifactId}.jnilib + + + + link-native-lib + generate-resources + + exec + + + ln + -sf ${MXNET_DIR}/lib/libmxnet.so ${project.build.directory}/libmxnet.so + + + + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 76bf00b54ba6..cf540d38bfd8 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -39,6 +39,8 @@ 2.11.8 2.11 + g++ + $ pom