Merge pull request #5 from daveliepmann/new-bert-example-with-finetun…

…ing-csv-fix Clojure BERT finetuning example: fix CSV parsing
gigasquid · May 1, 2019 · bb37715 · bb37715
2 parents d062d46 + 21178b4
commit bb37715
Showing 1 changed file with 12 additions and 12 deletions.
diff --git a/contrib/clojure-package/examples/bert/fine-tune-bert.ipynb b/contrib/clojure-package/examples/bert/fine-tune-bert.ipynb
@@ -114,8 +114,6 @@
    ],
    "source": [
     "(def model-path-prefix \"data/static_bert_base_net\")\n",
-    ";; epoch number of the model\n",
-    "(def epoch 0)\n",
     ";; the vocabulary used in the model\n",
     "(def vocab (bert-util/get-vocab))\n",
     ";; the input question\n",
@@ -217,28 +215,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "collapsed": false
-   },
+   "execution_count": 9,
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "He said the foodservice pie business doesn 't fit the company 's long-term growth strategy .\n",
-      " 100 percent behind George Bush \" and looked forward to using his years of training in the war .\n",
+      " The foodservice pie business does not fit our long-term growth strategy .\n",
       "1\n"
      ]
     }
    ],
    "source": [
-    "(def raw-file (csv/parse-csv (slurp \"data/dev.tsv\") :delimiter \\tab))\n",
+    "(def raw-file \n",
+    "    (csv/parse-csv (string/replace (slurp \"data/dev.tsv\") \"\\\"\" \"\")\n",
+    "                   :delimiter \\tab\n",
+    "                   :strict true))\n",
+    "\n",
     "(def data-train-raw (->> raw-file\n",
-    "                            (mapv #(vals (select-keys % [3 4 0])))\n",
-    "                            (rest) ;;drop header\n",
-    "                            (into [])\n",
-    "                            ))\n",
+    "                         (mapv #(vals (select-keys % [3 4 0])))\n",
+    "                         (rest) ; drop header\n",
+    "                         (into [])))\n",
+    "\n",
     "(def sample (first data-train-raw))\n",
     "(println (nth sample 0)) ;;;sentence a\n",
     "(println (nth sample 1)) ;; sentence b\n",