dataset(aslg_pc12): initial loading script

datasets/aslg_pc12/aslg_pc12.py

@@ -0,0 +1,85 @@
+# coding=utf-8
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ASLG-PC12: Synthetic English-ASL Gloss Parallel Corpus 2012"""
+
+from __future__ import absolute_import, division, print_function
+
+import csv
+import os
+
+import datasets
+
+_DESCRIPTION = """\
+Synthetic English-ASL Gloss Parallel Corpus 2012
+"""
+
+_CITATION = """\
+@inproceedings{othman2012english,
+  title={English-asl gloss parallel corpus 2012: Aslg-pc12},
+  author={Othman, Achraf and Jemni, Mohamed},
+  booktitle={5th Workshop on the Representation and Processing of Sign Languages: Interactions between Corpus and Lexicon LREC},
+  year={2012}
+}
+"""
+
+_GLOSS_URL = "https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.asl"
+_TEXT_URL = "https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.en"
+
+_HOMEPAGE = "https://achrafothman.net/site/asl-smt/"
+
+
+class ASLGPC12(datasets.GeneratorBasedBuilder):
+    """ASLG-PC12: Synthetic English-ASL Gloss Parallel Corpus 2012"""
+
+    VERSION = datasets.Version("0.0.1")  # sample corpus
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            # This defines the different columns of the dataset and their types
+            features=datasets.Features({
+                "gloss": datasets.Value("string"),  # American sign language gloss
+                "text": datasets.Value("string")  # English text
+            }),
+            homepage=_HOMEPAGE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        """Returns SplitGenerators."""
+
+        gloss_path, text_path = dl_manager.download([_GLOSS_URL, _TEXT_URL])
+
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                gen_kwargs={"gloss_path": gloss_path, "text_path": text_path},
+            )
+        ]
+
+    def _generate_examples(self, gloss_path, text_path):
+        """ Yields examples. """
+
+        gloss_f = open(gloss_path, "r")
+        text_f = open(text_path, "r")
+
+        for i, (gloss, text) in enumerate(zip(gloss_f, text_f)):
+            yield i, {
+                "gloss": gloss,
+                "text": text
+            }
+
+        gloss_f.close()
+        text_f.close()

datasets/aslg_pc12/dataset_infos.json

@@ -0,0 +1,53 @@
+{
+  "default": {
+    "description": "Synthetic English-ASL Gloss Parallel Corpus 2012\n",
+    "citation": "@inproceedings{othman2012english,\n  title={English-asl gloss parallel corpus 2012: Aslg-pc12},\n  author={Othman, Achraf and Jemni, Mohamed},\n  booktitle={5th Workshop on the Representation and Processing of Sign Languages: Interactions between Corpus and Lexicon LREC},\n  year={2012}\n}\n",
+    "homepage": "https://achrafothman.net/site/asl-smt/",
+    "license": "",
+    "features": {
+      "gloss": {
+        "dtype": "string",
+        "id": null,
+        "_type": "Value"
+      },
+      "text": {
+        "dtype": "string",
+        "id": null,
+        "_type": "Value"
+      }
+    },
+    "post_processed": null,
+    "supervised_keys": null,
+    "builder_name": "aslgp_c12",
+    "config_name": "default",
+    "version": {
+      "version_str": "0.0.1",
+      "description": null,
+      "major": 0,
+      "minor": 0,
+      "patch": 1
+    },
+    "splits": {
+      "train": {
+        "name": "train",
+        "num_bytes": 13497111,
+        "num_examples": 87710,
+        "dataset_name": "aslgp_c12"
+      }
+    },
+    "download_checksums": {
+      "https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.asl": {
+        "num_bytes": 6457824,
+        "checksum": "e550698336419edaaedfc51882541eb033db83257fa2167900ac0db9fae91633"
+      },
+      "https://www.achrafothman.net/aslsmt/corpus/sample-corpus-asl-en.en": {
+        "num_bytes": 6315607,
+        "checksum": "889732ced6ceaeab352efd647de65ac29e1ac7adb1206b2703d9d35432924f6a"
+      }
+    },
+    "download_size": 12773431,
+    "post_processing_size": null,
+    "dataset_size": 13497111,
+    "size_in_bytes": 26270542
+  }
+}