diff --git a/contrib/clojure-package/examples/cnn-text-classification/src/cnn_text_classification/classifier.clj b/contrib/clojure-package/examples/cnn-text-classification/src/cnn_text_classification/classifier.clj index 29ff36fe1ec0..94fd4f518c60 100644 --- a/contrib/clojure-package/examples/cnn-text-classification/src/cnn_text_classification/classifier.clj +++ b/contrib/clojure-package/examples/cnn-text-classification/src/cnn_text_classification/classifier.clj @@ -16,7 +16,9 @@ ;; (ns cnn-text-classification.classifier - (:require [cnn-text-classification.data-helper :as data-helper] + (:require [clojure.java.io :as io] + [clojure.java.shell :refer [sh]] + [cnn-text-classification.data-helper :as data-helper] [org.apache.clojure-mxnet.eval-metric :as eval-metric] [org.apache.clojure-mxnet.io :as mx-io] [org.apache.clojure-mxnet.module :as m] @@ -26,12 +28,18 @@ [org.apache.clojure-mxnet.context :as context]) (:gen-class)) +(def data-dir "data/") (def mr-dataset-path "data/mr-data") ;; the MR polarity dataset path (def glove-file-path "data/glove/glove.6B.50d.txt") (def num-filter 100) (def num-label 2) (def dropout 0.5) + + +(when-not (.exists (io/file (str data-dir))) + (do (println "Retrieving data for cnn text classification...") (sh "./get_data.sh"))) + (defn shuffle-data [test-num {:keys [data label sentence-count sentence-size embedding-size]}] (println "Shuffling the data and splitting into training and test sets") (println {:sentence-count sentence-count @@ -103,10 +111,10 @@ ;;; omit max-examples if you want to run all the examples in the movie review dataset ;; to limit mem consumption set to something like 1000 and adjust test size to 100 (println "Running with context devices of" devs) - (train-convnet {:devs [(context/cpu)] :embedding-size 50 :batch-size 10 :test-size 100 :num-epoch 10 :max-examples 1000}) + (train-convnet {:devs devs :embedding-size 50 :batch-size 10 :test-size 100 :num-epoch 10 :max-examples 1000}) ;; runs all the examples #_(train-convnet {:embedding-size 50 :batch-size 100 :test-size 1000 :num-epoch 10}))) (comment - (train-convnet {:devs [(context/cpu)] :embedding-size 50 :batch-size 10 :test-size 100 :num-epoch 10 :max-examples 1000})) + (train-convnet {:devs devs :embedding-size 50 :batch-size 10 :test-size 100 :num-epoch 10 :max-examples 1000})) diff --git a/contrib/clojure-package/examples/cnn-text-classification/test/cnn_text_classification/classifier_test.clj b/contrib/clojure-package/examples/cnn-text-classification/test/cnn_text_classification/classifier_test.clj new file mode 100644 index 000000000000..918a46f474d8 --- /dev/null +++ b/contrib/clojure-package/examples/cnn-text-classification/test/cnn_text_classification/classifier_test.clj @@ -0,0 +1,44 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns cnn-text-classification.classifier-test + (:require + [clojure.test :refer :all] + [org.apache.clojure-mxnet.module :as module] + [org.apache.clojure-mxnet.ndarray :as ndarray] + [org.apache.clojure-mxnet.util :as util] + [org.apache.clojure-mxnet.context :as context] + [cnn-text-classification.classifier :as classifier])) + +; +; The one and unique classifier test +; +(deftest classifier-test + (let [train + (classifier/train-convnet + {:devs [(context/default-context)] + :embedding-size 50 + :batch-size 10 + :test-size 100 + :num-epoch 1 + :max-examples 1000})] + (is (= ["data"] (util/scala-vector->vec (module/data-names train)))) + (is (= 20 (count (ndarray/->vec (-> train module/outputs first first))))))) + ;(prn (util/scala-vector->vec (data-shapes train))) + ;(prn (util/scala-vector->vec (label-shapes train))) + ;(prn (output-names train)) + ;(prn (output-shapes train)) \ No newline at end of file diff --git a/contrib/clojure-package/examples/gan/src/gan/gan_mnist.clj b/contrib/clojure-package/examples/gan/src/gan/gan_mnist.clj index e2e3364535ec..944791bce604 100644 --- a/contrib/clojure-package/examples/gan/src/gan/gan_mnist.clj +++ b/contrib/clojure-package/examples/gan/src/gan/gan_mnist.clj @@ -157,7 +157,9 @@ (save-img-diff i n calc-diff)))) -(defn train [devs] +(defn train + ([devs] (train devs num-epoch)) + ([devs num-epoch] (let [mod-d (-> (m/module (discriminator) {:contexts devs :data-names ["data"] :label-names ["label"]}) (m/bind {:data-shapes (mx-io/provide-data-desc mnist-iter) :label-shapes (mx-io/provide-label-desc mnist-iter) @@ -203,7 +205,7 @@ (save-img-gout i n (ndarray/copy (ffirst out-g))) (save-img-data i n batch) (calc-diff i n (ffirst diff-d))) - (inc n))))))) + (inc n)))))))) (defn -main [& args] (let [[dev dev-num] args diff --git a/contrib/clojure-package/examples/gan/test/gan/gan_test.clj b/contrib/clojure-package/examples/gan/test/gan/gan_test.clj new file mode 100644 index 000000000000..71b9126cae25 --- /dev/null +++ b/contrib/clojure-package/examples/gan/test/gan/gan_test.clj @@ -0,0 +1,25 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns gan.gan_test + (:require + [gan.gan-mnist :refer :all] + [org.apache.clojure-mxnet.context :as context] + [clojure.test :refer :all])) + +(deftest check-pdf + (train [(context/cpu)] 1)) \ No newline at end of file diff --git a/contrib/clojure-package/examples/imclassification/src/imclassification/train_mnist.clj b/contrib/clojure-package/examples/imclassification/src/imclassification/train_mnist.clj index a43dc3b69bd9..e61e9ebf6fbb 100644 --- a/contrib/clojure-package/examples/imclassification/src/imclassification/train_mnist.clj +++ b/contrib/clojure-package/examples/imclassification/src/imclassification/train_mnist.clj @@ -32,7 +32,7 @@ (def batch-size 10) ;; the batch size (def optimizer (optimizer/sgd {:learning-rate 0.01 :momentum 0.0})) (def eval-metric (eval-metric/accuracy)) -(def num-epoch 5) ;; the number of training epochs +(def num-epoch 1) ;; the number of training epochs (def kvstore "local") ;; the kvstore type ;;; Note to run distributed you might need to complile the engine with an option set (def role "worker") ;; scheduler/server/worker @@ -82,7 +82,9 @@ (sym/fully-connected "fc3" {:data data :num-hidden 10}) (sym/softmax-output "softmax" {:data data}))) -(defn start [devs] +(defn start + ([devs] (start devs num-epoch)) + ([devs _num-epoch] (when scheduler-host (println "Initing PS enviornments with " envs) (kvstore-server/init envs)) @@ -94,14 +96,18 @@ (do (println "Starting Training of MNIST ....") (println "Running with context devices of" devs) - (let [mod (m/module (get-symbol) {:contexts devs})] - (m/fit mod {:train-data train-data + (let [_mod (m/module (get-symbol) {:contexts devs})] + (m/fit _mod {:train-data train-data :eval-data test-data - :num-epoch num-epoch + :num-epoch _num-epoch :fit-params (m/fit-params {:kvstore kvstore :optimizer optimizer - :eval-metric eval-metric})})) - (println "Finish fit")))) + :eval-metric eval-metric})}) + (println "Finish fit") + _mod + ) + + )))) (defn -main [& args] (let [[dev dev-num] args diff --git a/contrib/clojure-package/examples/imclassification/test/imclassification/train_mnist_test.clj b/contrib/clojure-package/examples/imclassification/test/imclassification/train_mnist_test.clj new file mode 100644 index 000000000000..7978bd36454a --- /dev/null +++ b/contrib/clojure-package/examples/imclassification/test/imclassification/train_mnist_test.clj @@ -0,0 +1,28 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns imclassification.train-mnist-test + (:require + [clojure.test :refer :all] + [clojure.java.io :as io] + [org.apache.clojure-mxnet.context :as context] + [org.apache.clojure-mxnet.module :as module] + [imclassification.train-mnist :as mnist])) + +(deftest mnist-two-epochs-test + (module/save-checkpoint (mnist/start [(context/cpu)] 2) {:prefix "target/test" :epoch 2}) + (is (= (slurp "test/test-0002.params") (slurp "target/test-0002.params")))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/module/test/mnist_mlp_test.clj b/contrib/clojure-package/examples/module/test/mnist_mlp_test.clj new file mode 100644 index 000000000000..5fbcdd3c0b39 --- /dev/null +++ b/contrib/clojure-package/examples/module/test/mnist_mlp_test.clj @@ -0,0 +1,29 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; +(ns mnist-mlp-test + (:require + [mnist-mlp :refer :all] + [org.apache.clojure-mxnet.context :as context] + [clojure.test :refer :all])) + +(deftest run-those-tests + (let [devs [(context/cpu)]] + (run-intermediate-level-api :devs devs) + (run-intermediate-level-api :devs devs :load-model-epoch (dec num-epoch)) + (run-high-level-api devs) + (run-prediction-iterator-api devs) + (run-predication-and-calc-accuracy-manually devs))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/multi-label/test/multi_label_test.clj b/contrib/clojure-package/examples/multi-label/test/multi_label_test.clj new file mode 100644 index 000000000000..446a84626e72 --- /dev/null +++ b/contrib/clojure-package/examples/multi-label/test/multi_label_test.clj @@ -0,0 +1,26 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns multi_label_test + (:require + [multi-label.core :as label] + [clojure.java.io :as io] + [org.apache.clojure-mxnet.context :as context] + [clojure.test :refer :all])) + +(deftest run-multi-label + (label/train [(context/cpu)])) \ No newline at end of file diff --git a/contrib/clojure-package/examples/neural-style/src/neural_style/core.clj b/contrib/clojure-package/examples/neural-style/src/neural_style/core.clj index fcf402f3466d..cbe365abed40 100644 --- a/contrib/clojure-package/examples/neural-style/src/neural_style/core.clj +++ b/contrib/clojure-package/examples/neural-style/src/neural_style/core.clj @@ -39,7 +39,7 @@ (def content-weight 5) ;; the weight for the content image (def blur-radius 1) ;; the blur filter radius (def output-dir "output") -(def lr 10) ;; the learning rate +(def lr 10.0) ;; the learning rate (def tv-weight 0.01) ;; the magnitude on the tv loss (def num-epochs 1000) (def num-channels 3) @@ -157,9 +157,10 @@ out (ndarray/* out tv-weight)] (sym/bind out ctx {"img" img "kernel" kernel})))) -(defn train [devs] - - (let [dev (first devs) +(defn train + ([devs] (train devs 20)) + ([devs n-epochs] + (let [dev (first devs) content-np (preprocess-content-image content-image max-long-edge) content-np-shape (mx-shape/->vec (ndarray/shape content-np)) style-np (preprocess-style-image style-image content-np-shape) @@ -212,7 +213,7 @@ tv-grad-executor (get-tv-grad-executor img dev tv-weight) eps 0.0 e 0] - (doseq [i (range 20)] + (doseq [i (range n-epochs)] (ndarray/set (:data model-executor) img) (-> (:executor model-executor) (executor/forward) @@ -237,8 +238,10 @@ (println "Epoch " i "relative change " eps) (when (zero? (mod i 2)) (save-image (ndarray/copy img) (str output-dir "/out_" i ".png") blur-radius true))) - - (ndarray/set old-img img)))) + (ndarray/set old-img img)) + ; (save-image (ndarray/copy img) (str output-dir "/final.png") 0 false) + ; (postprocess-image img) + ))) (defn -main [& args] ;;; Note this only works on cpu right now diff --git a/contrib/clojure-package/examples/neural-style/test/neural_style/vgg_19_test.clj b/contrib/clojure-package/examples/neural-style/test/neural_style/vgg_19_test.clj new file mode 100644 index 000000000000..44f786e09eb6 --- /dev/null +++ b/contrib/clojure-package/examples/neural-style/test/neural_style/vgg_19_test.clj @@ -0,0 +1,56 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns neural-style.vgg-19-test + (:require + [clojure.test :refer :all] + [mikera.image.core :as img] + [clojure.java.io :as io] + [org.apache.clojure-mxnet.ndarray :as ndarray] + [org.apache.clojure-mxnet.context :as context] + [neural-style.core :as neural])) + +(defn pic-to-ndarray-vec[path] + (-> path + img/load-image + neural/image->ndarray + ndarray/->vec)) + +(defn last-modified-check[x] + (let [t (- (System/currentTimeMillis) (.lastModified x)) ] + (if (> 10000 t) ; 10 seconds + x + (throw (Exception. (str "Generated File Too Old: (" t " ms) [" x "]")))))) + +(defn latest-pic-to-ndarray-vec[folder] + (->> folder + io/as-file + (.listFiles) + (sort-by #(.lastModified %)) + last + (last-modified-check) + (.getPath) + pic-to-ndarray-vec)) + +; +; The one and unique classifier test +; +(deftest vgg-19-test + (neural/train [(context/cpu)] 3) + (is (= + (latest-pic-to-ndarray-vec "output") + (pic-to-ndarray-vec "test/ref_out_2.png")))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/neural-style/test/ref_out_2.png b/contrib/clojure-package/examples/neural-style/test/ref_out_2.png new file mode 100644 index 000000000000..589cd8ec48d7 Binary files /dev/null and b/contrib/clojure-package/examples/neural-style/test/ref_out_2.png differ diff --git a/contrib/clojure-package/examples/profiler/src/profiler/core.clj b/contrib/clojure-package/examples/profiler/src/profiler/core.clj index e366c578c551..67ba0feb8a9b 100644 --- a/contrib/clojure-package/examples/profiler/src/profiler/core.clj +++ b/contrib/clojure-package/examples/profiler/src/profiler/core.clj @@ -27,9 +27,9 @@ (def profiler-mode "symbolic") ;; can be symbolic, imperative, api, mem (def output-path ".") ;; the profile file output directory (def profiler-name "profile-matmul-20iter.json") -(def iter-num 100) -(def begin-profiling-iter 50) -(def end-profiling-iter 70) +(def iter-num 5) +(def begin-profiling-iter 0) +(def end-profiling-iter 1) (def gpu? false) (defn run [] diff --git a/contrib/clojure-package/examples/profiler/test/core_test.clj b/contrib/clojure-package/examples/profiler/test/core_test.clj new file mode 100644 index 000000000000..92cc58f0892a --- /dev/null +++ b/contrib/clojure-package/examples/profiler/test/core_test.clj @@ -0,0 +1,32 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns core_test + (:require + [profiler.core :as profiler] + [clojure.java.io :as io] + [clojure.test :refer :all])) + +(defn count-lines[file] + (count (line-seq (io/reader (io/as-file file))))) + +(deftest run-profiler + (profiler/run) + (let [new-file (clojure.java.io/as-file profiler/profiler-name)] + (is (.exists new-file)) + (is (= (count-lines profiler/profiler-name) (count-lines (str "test/" profiler/profiler-name)))) + (is (> 10000 (- (System/currentTimeMillis) (.lastModified new-file)))))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/rnn/test/rnn/core_test.clj b/contrib/clojure-package/examples/rnn/test/rnn/core_test.clj new file mode 100644 index 000000000000..ac4f5014e18c --- /dev/null +++ b/contrib/clojure-package/examples/rnn/test/rnn/core_test.clj @@ -0,0 +1,27 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns rnn.core_test + (:require + [rnn.train-char-rnn] ; this will load the data files + [rnn.test-char-rnn :as rnn] + [clojure.test :refer :all])) + +(deftest check-trained-network + (is (= + "The joke that we can start by the challenges of the American people. The American people have been talking about how to compete with the streets of San Antonio who the courage to come together as one " + (rnn/rnn-test "data/obama" 75 200 false)))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/tutorial/.gitignore b/contrib/clojure-package/examples/tutorial/.gitignore index c53038ec0e3d..338927e78384 100644 --- a/contrib/clojure-package/examples/tutorial/.gitignore +++ b/contrib/clojure-package/examples/tutorial/.gitignore @@ -9,3 +9,4 @@ pom.xml.asc /.nrepl-port .hgignore .hg/ +filename \ No newline at end of file diff --git a/contrib/clojure-package/examples/tutorial/project.clj b/contrib/clojure-package/examples/tutorial/project.clj index 8a78ec6a6abf..3bcfc0289644 100644 --- a/contrib/clojure-package/examples/tutorial/project.clj +++ b/contrib/clojure-package/examples/tutorial/project.clj @@ -19,6 +19,8 @@ :description "MXNET tutorials" :plugins [[lein-cljfmt "0.5.7"]] :dependencies [[org.clojure/clojure "1.9.0"] + [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.4.0-SNAPSHOT"] + ;; Uncomment the one appropriate for your machine & configuration: #_[org.apache.mxnet.contrib.clojure/clojure-mxnet-linux-cpu "1.4.0"] #_[org.apache.mxnet.contrib.clojure/clojure-mxnet-linux-gpu "1.4.0"] diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj index 4ca50ff5cd44..e19498111022 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj @@ -184,7 +184,7 @@ ])) (m/save-checkpoint mod {:prefix save-prefix :epoch epoch-num - :save-opt-states true}))) + :save-opt-states true}))) ;; INFO org.apache.mxnet.module.Module: Saved checkpoint to my-model-0000.params ;; INFO org.apache.mxnet.module.Module: Saved optimizer state to my-model-0000.states @@ -247,7 +247,40 @@ new-mod ;=> #object[org.apache.mxnet.module.Module 0x5304d0f4 "org.apache.mxnet. ;; Create `fit-params` and then use it to set `begin-epoch` so that ;; `fit` knows to resume from a saved epoch. + + +(comment +;; FIXME +; Caused by: java.io.EOFException +; at java.io.DataInputStream.readInt(DataInputStream.java:392) +; at java.io.ObjectInputStream$BlockDataInputStream.readInt(ObjectInputStream.java:3182) +; at java.io.ObjectInputStream.readInt(ObjectInputStream.java:1032) +; at org.apache.mxnet.Optimizer$$anon$1$$anonfun$deserializeState$1.apply$mcVI$sp(Optimizer.scala:84) +; at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160) +; at org.apache.mxnet.Optimizer$$anon$1.deserializeState(Optimizer.scala:83) +; at org.apache.mxnet.module.Module$$anonfun$loadOptimizerStates$3.apply(Module.scala:594) +; at org.apache.mxnet.module.Module$$anonfun$loadOptimizerStates$3.apply(Module.scala:589) +; at scala.Option.foreach(Option.scala:257) +; at org.apache.mxnet.module.Module.loadOptimizerStates(Module.scala:589) +; at org.apache.mxnet.module.Module$$anonfun$initOptimizer$4.apply(Module.scala:407) +; at org.apache.mxnet.module.Module$$anonfun$initOptimizer$4.apply(Module.scala:406) +; at scala.Option.foreach(Option.scala:257) +; at org.apache.mxnet.module.Module.initOptimizer(Module.scala:406) +; at org.apache.mxnet.module.BaseModule.fit(BaseModule.scala:407) +; at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) +; at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) +; at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) +; at java.lang.reflect.Method.invoke(Method.java:498) +; at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) +; at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28) +; at org.apache.clojure_mxnet.module$fit.invokeStatic(module.clj:551) +; at org.apache.clojure_mxnet.module$fit.invoke(module.clj:538) +; at tutorial.module$eval1787.invokeStatic(module.clj:250) +; at tutorial.module$eval1787.invoke(module.clj:250) + (m/fit new-mod {:train-data train-data :eval-data test-data :num-epoch 2 :fit-params (m/fit-params {:begin-epoch 1})}) + +) \ No newline at end of file diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj index 8e51de215157..d18bb53daaf1 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj @@ -91,8 +91,8 @@ (ndarray/save "filename" {"arr1" arr1 "arr2" arr2}) ;; (you can also do "s3://path" or "hdfs") -(ndarray/save "/Users/daveliepmann/src/coursework/mxnet-clj-tutorials/abc" - {"arr1" arr1 "arr2" arr2}) +;; (ndarray/save "/Users/daveliepmann/src/coursework/mxnet-clj-tutorials/abc" +;; {"arr1" arr1 "arr2" arr2}) ;; To load: (def from-file (ndarray/load "filename")) @@ -114,7 +114,9 @@ from-file ;=>{"arr1" #object[org.apache.mxnet.NDArray 0x6115ba61 "org.apache.mxn (def cpu-a (ndarray/zeros [100 200])) (ndarray/context cpu-a) ;=> #object[org.apache.mxnet.Context 0x3f376123 "cpu(0)"] -(def gpu-b (ndarray/zeros [100 200] {:ctx (context/gpu 0)})) ;; to use with gpu +(comment + (def gpu-b (ndarray/zeros [100 200] {:ctx (context/gpu 0)})) ;; to use with gpu +) ;; Currently, we do not allow operations among arrays from different ;; contexts. To manually enable this, use the `copy-to` function to diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj index ebf4f7e96797..e88260069015 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj @@ -125,7 +125,9 @@ net ;=> #object[org.apache.mxnet.Symbol 0x5c78c8c2 "org.apache.mxnet.Symbol@5c78 (first) (ndarray/->vec));=> [2.0 2.0 2.0 2.0] -;; We can evaluate the same symbol on GPU with different data. -;; (To do this you must have the correct native library jar defined as a dependency.) -(def ex (sym/bind c (context/gpu 0) {"a" (ndarray/ones [2 2]) - "b" (ndarray/ones [2 2])})) +(comment + ;; We can evaluate the same symbol on GPU with different data. + ;; (To do this you must have the correct native library jar defined as a dependency.) + (def ex (sym/bind c (context/gpu 0) {"a" (ndarray/ones [2 2]) + "b" (ndarray/ones [2 2])})) +) diff --git a/contrib/clojure-package/examples/tutorial/test/tutorial/core_test.clj b/contrib/clojure-package/examples/tutorial/test/tutorial/core_test.clj new file mode 100644 index 000000000000..0e5169c5cfaa --- /dev/null +++ b/contrib/clojure-package/examples/tutorial/test/tutorial/core_test.clj @@ -0,0 +1,27 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns tutorial.core_test + (:require [clojure.test :refer :all]) + (:require + [tutorial.introduction] + [tutorial.kvstore] + [tutorial.module] + [tutorial.ndarray] + [tutorial.symbol])) + +(deftest if-this-goes-here-then-tutorials-have-loaded-properly (is true)) \ No newline at end of file diff --git a/contrib/clojure-package/examples/visualization/test/visualization/core_test.clj b/contrib/clojure-package/examples/visualization/test/visualization/core_test.clj new file mode 100644 index 000000000000..1b10695cb34c --- /dev/null +++ b/contrib/clojure-package/examples/visualization/test/visualization/core_test.clj @@ -0,0 +1,28 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns visualization.core_test + (:require + [visualization.core :as visualization] + [clojure.test :refer :all])) + +(deftest check-pdf + (visualization/test-viz) + (let [new-pdf (clojure.java.io/as-file "testviz.pdf")] + (is (.exists new-pdf)) + (is (> 10000 (- (System/currentTimeMillis) (.lastModified new-pdf)))))) + \ No newline at end of file diff --git a/contrib/clojure-package/integration-tests.sh b/contrib/clojure-package/integration-tests.sh new file mode 100755 index 000000000000..28f21c8fc482 --- /dev/null +++ b/contrib/clojure-package/integration-tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -evx + +MXNET_HOME=${PWD} +EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples +#cd ${MXNET_HOME}/contrib/clojure-package +#lein test +#lein cloverage --codecov +for i in `ls ${EXAMPLES_HOME}` ; do +cd ${EXAMPLES_HOME}/${i} && lein test +done