From 1a7452515d42b1077fb0485f876c1d9263e188a9 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Fri, 29 Mar 2019 11:45:10 -0400 Subject: [PATCH 1/4] add test for drawing bounding box --- .../test/org/apache/clojure_mxnet/image_test.clj | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj index 38ab11c86012..b638e8b6a69f 100644 --- a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj +++ b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj @@ -20,7 +20,8 @@ [org.apache.clojure-mxnet.ndarray :as ndarray] [clojure.java.io :as io] [clojure.test :refer :all]) - (:import (javax.imageio ImageIO))) + (:import (javax.imageio ImageIO) + (java.io File))) (def tmp-dir (System/getProperty "java.io.tmpdir")) (def image-path (.getAbsolutePath (io/file tmp-dir "Pug-Cookie.jpg"))) @@ -76,4 +77,14 @@ (let [img-arr (image/read-image image-path) resized-arr (image/resize-image img-arr 224 224) new-img (image/to-image resized-arr)] - (is (= true (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) + (is (true? (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) + +(deftest test-draw-bounding-box! + (let [new-img (-> (ImageIO/read (new File image-path)) + (image/draw-bounding-box! [{:xmin 190 :xmax 850 :ymin 50 :ymax 450} + {:xmin 200 :xmax 350 :ymin 440 :ymax 530}] + {:stroke 2 + :names ["pug" "cookie"] + :transparency 0.8 + :font-size-mult 2.0}))] + (is (true? (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) From 32bdf6cc09efa29705d4b199f7b5ea54dbb033e3 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Fri, 29 Mar 2019 13:14:51 -0400 Subject: [PATCH 2/4] Uses the core image drawing bounding box functionality for the object detection example. Adjust the specs and names to make it easier to run with object detection and clojure draw bounding box example --- .../examples/infer/objectdetector/project.clj | 1 - .../infer/objectdetector/src/infer/draw.clj | 44 --------------- .../src/infer/objectdetector_example.clj | 55 ++++++++++--------- contrib/clojure-package/integration-tests.sh | 2 +- .../src/org/apache/clojure_mxnet/image.clj | 26 ++++----- .../org/apache/clojure_mxnet/image_test.clj | 4 +- 6 files changed, 46 insertions(+), 86 deletions(-) delete mode 100644 contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj diff --git a/contrib/clojure-package/examples/infer/objectdetector/project.clj b/contrib/clojure-package/examples/infer/objectdetector/project.clj index cdd9a8991dc8..da01797f5a21 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/project.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/project.clj @@ -22,7 +22,6 @@ :aliases {"run-detector" ["run" "--" "-m" "models/resnet50_ssd/resnet50_ssd_model" "-i" "images/dog.jpg" "-d" "images/"]} :dependencies [[org.clojure/clojure "1.9.0"] [org.clojure/tools.cli "0.4.1"] - [origami "4.0.0-3"] [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.5.0-SNAPSHOT"]] :main ^:skip-aot infer.objectdetector-example :profiles {:uberjar {:aot :all}}) diff --git a/contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj b/contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj deleted file mode 100644 index d29b34b5c22a..000000000000 --- a/contrib/clojure-package/examples/infer/objectdetector/src/infer/draw.clj +++ /dev/null @@ -1,44 +0,0 @@ -;; Licensed to the Apache Software Foundation (ASF) under one or more -;; contributor license agreements. See the NOTICE file distributed with -;; this work for additional information regarding copyright ownership. -;; The ASF licenses this file to You under the Apache License, Version 2.0 -;; (the "License"); you may not use this file except in compliance with -;; the License. You may obtain a copy of the License at -;; -;; http://www.apache.org/licenses/LICENSE-2.0 -;; -;; Unless required by applicable law or agreed to in writing, software -;; distributed under the License is distributed on an "AS IS" BASIS, -;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -;; See the License for the specific language governing permissions and -;; limitations under the License. -;; - -(ns infer.draw - (:require - [opencv4.colors.rgb :as rgb] - [opencv4.core :refer [FONT_HERSHEY_PLAIN imread imwrite new-point put-text! rectangle]])) - -(defn black-boxes! [img results] - (doseq [{confidence :confidence label :label top-left :top-left bottom-right :bottom-right} results] - (let [w (.width img) - h (.height img) - top-left-p (new-point (int (* w (first top-left))) (int (* h (second top-left)))) - bottom-right-p (new-point (int (* w (first bottom-right))) (int (* h (second bottom-right))))] - (if (< 15 confidence) - (do - (rectangle img top-left-p bottom-right-p rgb/white 1) - (put-text! img - (str label "[" confidence "% ]") - top-left-p - FONT_HERSHEY_PLAIN - 1.0 - rgb/white 1))))) - img) - -(defn draw-bounds [image results output-dir] - (let [out-file (str output-dir "/" (.getName (clojure.java.io/as-file image)))] - (-> image - (imread) - (black-boxes! results) - (imwrite out-file)))) \ No newline at end of file diff --git a/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj b/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj index 9331798b038c..9361641b5693 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj @@ -17,13 +17,15 @@ (ns infer.objectdetector-example (:require [org.apache.clojure-mxnet.context :as context] [org.apache.clojure-mxnet.dtype :as dtype] + [org.apache.clojure-mxnet.image :as image] [org.apache.clojure-mxnet.infer :as infer] [org.apache.clojure-mxnet.layout :as layout] [clojure.java.io :as io] - [infer.draw :as draw] - [clojure.string :refer [join]] + [clojure.string :as string] [clojure.tools.cli :refer [parse-opts]]) - (:gen-class)) + (:gen-class) + (:import (javax.imageio ImageIO) + (java.io File))) (defn check-valid-dir "Check that the input directory exists" @@ -54,27 +56,30 @@ :validate [check-valid-dir "Input directory not found"]] ["-h" "--help"]]) -(defn result->map [{:keys [class prob x-min y-min x-max y-max]}] - (hash-map - :label class - :confidence (int (* 100 prob)) - :top-left [x-min y-min] - :bottom-right [x-max y-max])) - -(defn print-results [results] - (doseq [_r results] - (println (format "Class: %s Confidence=%s Coords=(%s, %s)" - (_r :label) - (_r :confidence) - (_r :top-left) - (_r :bottom-right))))) - (defn process-results [images results output-dir] - (dotimes [i (count images)] - (let [image (nth images i) _results (map result->map (nth results i))] - (println "processing: " image) - (print-results _results) - (draw/draw-bounds image _results output-dir)))) + (mapv (fn [image-path result] + (println "looking at image" image-path) + (println "result: " result) + (let [buf (ImageIO/read (new File image-path)) + width (.getWidth buf) + height (.getHeight buf) + names (mapv :class result) + coords (mapv (fn [result] + (-> result + (update :x-min #(* width %)) + (update :x-max #(* width %)) + (update :y-min #(* height %)) + (update :y-max #(* height %)))) + result) + new-img (-> (ImageIO/read (new File image-path)) + (image/draw-bounding-box! coords + {:stroke 2 + :names (mapv #(str (:class %) "-" (:prob %)) + result) + :transparency 0.5 + :font-size-mult 1.0}))] + (ImageIO/write new-img "jpg" (io/file output-dir (last (string/split image-path #"\/")))))) + images results)) (defn detect-single-image "Detect objects in a single image and print top-5 predictions" @@ -109,7 +114,7 @@ (apply concat (for [image-files image-file-batches] (let [image-batch (infer/load-image-paths image-files) - topk 5 + topk 5 res (infer/detect-objects-batch detector image-batch topk) ] (process-results image-files @@ -143,5 +148,5 @@ (parse-opts args cli-options)] (cond (:help options) (println summary) - (some? errors) (println (join "\n" errors)) + (some? errors) (println (string/join "\n" errors)) :else (run-detector options)))) diff --git a/contrib/clojure-package/integration-tests.sh b/contrib/clojure-package/integration-tests.sh index 5ae26e8dfcd4..3df9ba9787b5 100755 --- a/contrib/clojure-package/integration-tests.sh +++ b/contrib/clojure-package/integration-tests.sh @@ -26,7 +26,7 @@ lein install # then run through the examples EXAMPLES_HOME=${MXNET_HOME}/contrib/clojure-package/examples # use AWK pattern for blacklisting -TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification|gan|neural-style|infer|pre-trained-models/'` +TEST_CASES=`find ${EXAMPLES_HOME} -name test | awk '!/dontselect1|cnn-text-classification|gan|neural-style|pre-trained-models/'` for i in $TEST_CASES ; do cd ${i} && lein test done diff --git a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj index e2e98c4e8f01..f81a35803171 100644 --- a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj +++ b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj @@ -202,11 +202,11 @@ (Image/toImage input)) (s/def ::buffered-image #(instance? BufferedImage %)) -(s/def ::xmin integer?) -(s/def ::xmax integer?) -(s/def ::ymin integer?) -(s/def ::ymax integer?) -(s/def ::coordinate (s/keys :req-un [::xmin ::xmax ::ymin ::ymax])) +(s/def ::x-min number?) +(s/def ::x-max number?) +(s/def ::y-min number?) +(s/def ::y-max number?) +(s/def ::coordinate (s/keys :req-un [::x-min ::x-max ::y-min ::y-max])) (s/def ::coordinates (s/coll-of ::coordinate)) (s/def ::names (s/nilable (s/coll-of string?))) (s/def ::stroke (s/and integer? pos?)) @@ -217,11 +217,11 @@ (defn- convert-coordinate "Convert bounding box coordinate to Scala correct types." - [{:keys [xmin xmax ymin ymax]}] - {:xmin (int xmin) - :xmax (int xmax) - :ymin (int ymin) - :ymax (int ymax)}) + [{:keys [x-min x-max y-min y-max]}] + {:xmin (int x-min) + :xmax (int x-max) + :ymin (int y-min) + :ymax (int y-max)}) (defn draw-bounding-box! "Draw bounding boxes on `buffered-image` and Mutate the input image. @@ -233,9 +233,9 @@ `transparency`: float in (0.0, 1.0) - Transparency of the bounding box returns: Modified `buffered-image` Ex: - (draw-bounding-box! img [{:xmin 0 :xmax 100 :ymin 0 :ymax 100}]) - (draw-bounding-box! [{:xmin 190 :xmax 850 :ymin 50 :ymax 450} - {:xmin 200 :xmax 350 :ymin 440 :ymax 530}] + (draw-bounding-box! img [{:x-min 0 :x-max 100 :y-min 0 :y-max 100}]) + (draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450} + {:x-min 200 :x-max 350 :y-min 440 :y-max 530}] {:stroke 2 :names [\"pug\" \"cookie\"] :transparency 0.8 diff --git a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj index b638e8b6a69f..aec9e27ced04 100644 --- a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj +++ b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj @@ -81,8 +81,8 @@ (deftest test-draw-bounding-box! (let [new-img (-> (ImageIO/read (new File image-path)) - (image/draw-bounding-box! [{:xmin 190 :xmax 850 :ymin 50 :ymax 450} - {:xmin 200 :xmax 350 :ymin 440 :ymax 530}] + (image/draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450} + {:x-min 200 :x-max 350 :y-min 440 :y-max 530}] {:stroke 2 :names ["pug" "cookie"] :transparency 0.8 From 5fec0cf96e8cfd3e07adb69085419de1dc8327c6 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Wed, 10 Apr 2019 17:51:50 -0400 Subject: [PATCH 3/4] feedback from @chouffe --- .../src/infer/objectdetector_example.clj | 56 ++++++++++--------- .../org/apache/clojure_mxnet/image_test.clj | 7 ++- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj b/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj index 9361641b5693..65d822ff36aa 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/src/infer/objectdetector_example.clj @@ -56,30 +56,36 @@ :validate [check-valid-dir "Input directory not found"]] ["-h" "--help"]]) + +(defn process-result! [output-dir image-path predictions] + (println "looking at image" image-path) + (println "predictions: " predictions) + (let [buf (ImageIO/read (new File image-path)) + width (.getWidth buf) + height (.getHeight buf) + names (mapv :class predictions) + coords (mapv (fn [prediction] + (-> prediction + (update :x-min #(* width %)) + (update :x-max #(* width %)) + (update :y-min #(* height %)) + (update :y-max #(* height %)))) + predictions) + new-img (-> (ImageIO/read (new File image-path)) + (image/draw-bounding-box! coords + {:stroke 2 + :names (mapv #(str (:class %) "-" (:prob %)) + predictions) + :transparency 0.5 + + :font-size-mult 1.0}))] + (->> (string/split image-path #"\/") + last + (io/file output-dir) + (ImageIO/write new-img "jpg")))) + (defn process-results [images results output-dir] - (mapv (fn [image-path result] - (println "looking at image" image-path) - (println "result: " result) - (let [buf (ImageIO/read (new File image-path)) - width (.getWidth buf) - height (.getHeight buf) - names (mapv :class result) - coords (mapv (fn [result] - (-> result - (update :x-min #(* width %)) - (update :x-max #(* width %)) - (update :y-min #(* height %)) - (update :y-max #(* height %)))) - result) - new-img (-> (ImageIO/read (new File image-path)) - (image/draw-bounding-box! coords - {:stroke 2 - :names (mapv #(str (:class %) "-" (:prob %)) - result) - :transparency 0.5 - :font-size-mult 1.0}))] - (ImageIO/write new-img "jpg" (io/file output-dir (last (string/split image-path #"\/")))))) - images results)) + (doall (map (partial process-result! output-dir) images results))) (defn detect-single-image "Detect objects in a single image and print top-5 predictions" @@ -87,7 +93,7 @@ ([detector input-image output-dir] (.mkdir (io/file output-dir)) (let [image (infer/load-image-from-file input-image) - topk 5 + topk 3 res (infer/detect-objects detector image topk) ] (process-results @@ -114,7 +120,7 @@ (apply concat (for [image-files image-file-batches] (let [image-batch (infer/load-image-paths image-files) - topk 5 + topk 3 res (infer/detect-objects-batch detector image-batch topk) ] (process-results image-files diff --git a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj index aec9e27ced04..23b88d07e896 100644 --- a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj +++ b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj @@ -77,14 +77,15 @@ (let [img-arr (image/read-image image-path) resized-arr (image/resize-image img-arr 224 224) new-img (image/to-image resized-arr)] - (is (true? (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) + (is (ImageIO/write new-img "png" (io/file tmp-dir "out.png"))))) (deftest test-draw-bounding-box! - (let [new-img (-> (ImageIO/read (new File image-path)) + (let [orig-img (ImageIO/read (new File image-path)) + new-img (-> orig-img (image/draw-bounding-box! [{:x-min 190 :x-max 850 :y-min 50 :y-max 450} {:x-min 200 :x-max 350 :y-min 440 :y-max 530}] {:stroke 2 :names ["pug" "cookie"] :transparency 0.8 :font-size-mult 2.0}))] - (is (true? (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) + (is (ImageIO/write new-img "png" (io/file tmp-dir "out.png"))))) From 5e18d8ee24448314545347db96cfc5d2e4c38847 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Wed, 10 Apr 2019 17:52:17 -0400 Subject: [PATCH 4/4] refactor to be 3 top predictions instead of 5 to make the images less crowded --- .../test/infer/objectdetector_example_test.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj b/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj index 696d96b3ae3a..3d20c614918f 100644 --- a/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj +++ b/contrib/clojure-package/examples/infer/objectdetector/test/infer/objectdetector_example_test.clj @@ -47,11 +47,11 @@ {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)] (clojure.pprint/pprint predictions) (is (some? predictions)) - (is (= 5 (count predictions))) + (is (= 3 (count predictions))) (is (string? class)) (is (< 0.8 prob)) (is (every? #(< 0 % 1) [x-min x-max y-min y-max])) - (is (= #{"dog" "person" "bicycle" "car"} (set (mapv :class predictions)))))) + (is (= #{"dog" "bicycle" "car"} (set (mapv :class predictions)))))) (deftest test-batch-detection (let [detector (create-detector) @@ -60,7 +60,7 @@ predictions (first batch-predictions) {:keys [class prob x-min x-max y-min y-max] :as pred} (first predictions)] (is (some? batch-predictions)) - (is (= 5 (count predictions))) + (is (= 3 (count predictions))) (is (string? class)) (is (< 0.8 prob)) (println [x-min x-max y-min y-max])