Further update task domains & categories; Add Utility Scripts

README.md

@@ -68,6 +68,7 @@ We would appreciate any external contributions! 🙏
     * If you're building your tasks based existing datasets and their crowdsourcing templates, see these [guidelines](doc/crowdsourcing.md). 
  * Add your task to [our list of tasks](tasks/README.md).
  * To make sure that your addition is formatted correctly, run the tests: `> python src/test_all.py`
+    * To only test the formatting of a range of tasks, run `> python src/test_all.py --task <begin_task_number> <end_task_number>`. For example, running `> python src/test_all.py --task 5 10` will run the test from task005 to task010.
 
 
 If you have any questions or suggestions, please use [the issues](/~https://github.com/allenai/natural-instructions-expansion/issues) feature.  

doc/task-hierarchy.md

@@ -8,6 +8,8 @@
   - `Answer Generation -> Fill in the Blank`
   - `Answer Generation -> Multiple Choice Question Answering`
   - `Answer Generation -> Open Question Answering`
+  - `Answer Generation -> Extractive`
+  - `Answer Generation -> Abstractive`
 - `Author Identification`
 - `Classification`
 - `Combinatorics`
@@ -23,7 +25,9 @@
 - `Ethical Judgement`
 - `Explanation Generation`
 - `Fake News Detection`
-- `Grammar Error Correction`
+- `Grammar Error`
+  -  `Grammar Error -> Grammar Error Correction`
+  -  `Grammar Error -> Grammar Error Detection`
 - `Hallucination`: Given a context, generate imaginary content e.g. given a sentence, generate a story/poem.
 - `Hate Speech Detection`
 - `Hypernym Discovery`

src/auto_add_domain.py

@@ -0,0 +1,48 @@
+import json
+import os
+from os import listdir, path
+from os.path import isfile, join
+
+"""
+Script for adding the "Domains" field for all the tasks of a given dataset.
+"""
+
+
+with open("doc/task-hierarchy.md", 'r') as readmef:
+    hierarchy_content = " ".join(readmef.readlines())
+
+tasks_path = 'tasks/'
+# modify these variables as needed
+dataset_name = "mctaco"
+domains = [
+        "News",
+        "Wikipedia",
+        "Law",
+        "Justice",
+        "History",
+        "History -> 9/11 Reports",
+        "Anthropology",
+        "School Science Textbooks",
+        "Fiction"
+    ]
+
+
+
+def add_domain(tasks_path, dataset_name, domains):
+    for d in domains: 
+        assert d in hierarchy_content, f"domain {d} not in task-hierarchy"
+    # find the task files containing the dataset
+    files = [join(tasks_path, f) for f in listdir(tasks_path) if isfile(join(tasks_path, f)) and dataset_name in f]
+    files.sort()
+    # add the domain
+    for file in files: 
+        with open(file, 'r') as f:
+            data = json.load(f)
+            data['Domains'] = domains
+        os.remove(file)
+        with open(file, 'w') as f:
+            modified_json = json.dumps(data, indent=4, ensure_ascii=False)
+            print(modified_json, file=f)
+
+if __name__ == "__main__":
+    add_domain(tasks_path, dataset_name, domains)

src/test_all.py

@@ -3,6 +3,22 @@
 import json
 from os import listdir
 from os.path import isfile, join
+import argparse
+
+# get the range of task you want to test, if specified in the command line
+parser = argparse.ArgumentParser()
+parser.add_argument("--task",
+                        nargs=2,
+                        type=int,
+                        required=False,
+                        help="The range of task you want to parse")
+
+args = parser.parse_args()
+if args.task:
+    begin_task_number, end_task_number = args.task[0], args.task[1]
+    assert begin_task_number > 0, "begin task must be greater than 0"
+    assert end_task_number > begin_task_number, "please specify a range of task you would like to test; i.e. the end task number must be greater than beginning task number"
+
 
 # make sure that there is no json file in the root directory 
 root_files = [f for f in listdir('.') if isfile(join('.', f))]
@@ -66,7 +82,11 @@ def assert_language_name(name):
     file_name = name + ".json"
     assert file_name in files, f" Did not find `{file_name}` among {files}"
 
-for file in files:
+# test every file (README is skipped)
+if not args.task:
+    begin_task_number, end_task_number = 1, len(files)
+
+for file in files[begin_task_number:end_task_number+1]:
     if ".json" in file:
         print(f" --> testing file: {file}")
         assert '.json' in file, 'the file does not seem to have a .json in it: ' + file

tasks/task003_mctaco_question_generation_event_duration.json

@@ -7,9 +7,11 @@
         "mctaco"
     ],
     "Categories": [
-        "Question Generation"
+        "Contextual Question Generation",
+        "Reasoning -> Temporal Reasoning",
+        "Reasoning -> Commonsense Reasoning"    
     ],
-    "Definition": "In this task, we ask you to write a question that involves \u201cevent duration\", based on a given sentence. Here, event duration is defined as the understanding of how long events typically last. For example, \u201cbrushing teeth\u201d, usually takes few minutes.\nThings to avoid: Don't create questions which have explicit mentions of answers in text. Instead, it has to be implied from what is given. In other words, we want you to use \"instinct\" or \"common sense\".\nEmphasis & Caution: The written questions are not required to have a single correct answer. ",
+    "Definition": "In this task, we ask you to write a question that involves “event duration\", based on a given sentence. Here, event duration is defined as the understanding of how long events typically last. For example, “brushing teeth”, usually takes few minutes.\nThings to avoid: Don't create questions which have explicit mentions of answers in text. Instead, it has to be implied from what is given. In other words, we want you to use \"instinct\" or \"common sense\".\nEmphasis & Caution: The written questions are not required to have a single correct answer. ",
     "Positive Examples": [
         {
             "input": "Sentence: Jack played basketball after school, after which he was very tired.",
@@ -301,7 +303,7 @@
             ]
         },
         {
-            "input": "Sentence: No, of course not\u2026 I\u2013  \"Then why,\" Arthur barreled on, \"do you disregard them all and return to this superstitious babble, this morbid, paganistic drivel?.",
+            "input": "Sentence: No, of course not… I–  \"Then why,\" Arthur barreled on, \"do you disregard them all and return to this superstitious babble, this morbid, paganistic drivel?.",
             "output": [
                 "How long has Arthur been talking?"
             ]
@@ -524,7 +526,7 @@
             ]
         },
         {
-            "input": "Sentence: Britain seized Gibraltar\u2002\u2014\u2002in the name of the Hapsburg claimant\u2002\u2014\u2002and retained it when the war was over.",
+            "input": "Sentence: Britain seized Gibraltar — in the name of the Hapsburg claimant — and retained it when the war was over.",
             "output": [
                 "How long did the war happen?"
             ]
@@ -1214,7 +1216,7 @@
             ]
         },
         {
-            "input": "Sentence: Sarawak's coast and jungle interior were controlled by the Iban\u2002\u2014\u2002Sea Dayak pirates and Land Dayak slash-and-burn farmers.",
+            "input": "Sentence: Sarawak's coast and jungle interior were controlled by the Iban — Sea Dayak pirates and Land Dayak slash-and-burn farmers.",
             "output": [
                 "How long did it take for the Iban to take control of Sarawak's coast?"
             ]
@@ -2084,7 +2086,7 @@
             ]
         },
         {
-            "input": "Sentence: Zarco and Teixeira were appointed co-go\u00advern\u00adors of Madeira, while Perestrelo was awarded Porto Santo.",
+            "input": "Sentence: Zarco and Teixeira were appointed co-go­vern­ors of Madeira, while Perestrelo was awarded Porto Santo.",
             "output": [
                 "How long were Zarco and Teixeira co-governors?"
             ]
@@ -2312,7 +2314,7 @@
             ]
         },
         {
-            "input": "Sentence: , followed by other ports at Hy\u00e8res, Antibes, and Nice.",
+            "input": "Sentence: , followed by other ports at Hyères, Antibes, and Nice.",
             "output": [
                 "How long were you at the ports?"
             ]
@@ -2668,5 +2670,16 @@
     ],
     "Instruction_language": [
         "English"
+    ],
+    "Domains": [
+        "News",
+        "Wikipedia",
+        "Law",
+        "Justice",
+        "History",
+        "History -> 9/11 Reports",
+        "Anthropology",
+        "School Science Textbooks",
+        "Fiction"
     ]
-}
+}

tasks/task004_mctaco_answer_generation_event_duration.json

@@ -7,9 +7,13 @@
         "mctaco"
     ],
     "Categories": [
-        "Answer Generation"
+        "Answer Generation -> Commonsense Question Answering",
+        "Reasoning -> Temporal Reasoning",
+        "Reasoning -> Commonsense Reasoning",
+        "Answer Generation -> Contextual Question Answering",
+        "Answer Generation -> Abstractive"
     ],
-    "Definition": "In this task we ask you to write answer to a question that involves \u201cevent duration\", based on a given sentence. Here, event duration is defined as the understanding of how long events typically last. For example, \u201cbrushing teeth\u201d, usually takes few minutes.\nThings to avoid: -\nEmphasis & Caution: Note that a lot of the questions could have more than one correct answers. We only need a single most-likely answer. Please try to keep your \"answer\" as simple as possible. Concise and simple \"answer\" is preferred over those complex and verbose ones.  ",
+    "Definition": "In this task we ask you to write answer to a question that involves “event duration\", based on a given sentence. Here, event duration is defined as the understanding of how long events typically last. For example, “brushing teeth”, usually takes few minutes.\nThings to avoid: -\nEmphasis & Caution: Note that a lot of the questions could have more than one correct answers. We only need a single most-likely answer. Please try to keep your \"answer\" as simple as possible. Concise and simple \"answer\" is preferred over those complex and verbose ones.  ",
     "Positive Examples": [
         {
             "input": "Sentence: Jack played basketball after school, after which he was very tired. \nQuestion: How long did Jack play basketball?",
@@ -1007,7 +1011,7 @@
             ]
         },
         {
-            "input": "Sentence: Britain seized Gibraltar\u2002\u2014\u2002in the name of the Hapsburg claimant\u2002\u2014\u2002and retained it when the war was over. \nQuestion: How long did the war happen?",
+            "input": "Sentence: Britain seized Gibraltar — in the name of the Hapsburg claimant — and retained it when the war was over. \nQuestion: How long did the war happen?",
             "output": [
                 "2 years.",
                 "a few years.",
@@ -1607,7 +1611,7 @@
             ]
         },
         {
-            "input": "Sentence: , followed by other ports at Hy\u00e8res, Antibes, and Nice. \nQuestion: How long were you at the ports?",
+            "input": "Sentence: , followed by other ports at Hyères, Antibes, and Nice. \nQuestion: How long were you at the ports?",
             "output": [
                 "four weeks.",
                 "four hours."
@@ -1859,7 +1863,7 @@
             ]
         },
         {
-            "input": "Sentence: No, of course not\u2026 I\u2013  \"Then why,\" Arthur barreled on, \"do you disregard them all and return to this superstitious babble, this morbid, paganistic drivel?. \nQuestion: How long has Arthur been talking?",
+            "input": "Sentence: No, of course not… I–  \"Then why,\" Arthur barreled on, \"do you disregard them all and return to this superstitious babble, this morbid, paganistic drivel?. \nQuestion: How long has Arthur been talking?",
             "output": [
                 "10 minutes.",
                 "a few minutes."
@@ -1902,7 +1906,7 @@
             ]
         },
         {
-            "input": "Sentence: Zarco and Teixeira were appointed co-go\u00advern\u00adors of Madeira, while Perestrelo was awarded Porto Santo. \nQuestion: How long were Zarco and Teixeira co-governors?",
+            "input": "Sentence: Zarco and Teixeira were appointed co-go­vern­ors of Madeira, while Perestrelo was awarded Porto Santo. \nQuestion: How long were Zarco and Teixeira co-governors?",
             "output": [
                 "four years.",
                 "zarco and teixeira were co-governors for a term of 4 years.",
@@ -2769,5 +2773,16 @@
     ],
     "Instruction_language": [
         "English"
+    ],
+    "Domains": [
+        "News",
+        "Wikipedia",
+        "Law",
+        "Justice",
+        "History",
+        "History -> 9/11 Reports",
+        "Anthropology",
+        "School Science Textbooks",
+        "Fiction"
     ]
-}
+}

tasks/task005_mctaco_wrong_answer_generation_event_duration.json

@@ -7,9 +7,13 @@
         "mctaco"
     ],
     "Categories": [
-        "Incorrect Answer Generation"
+        "Incorrect Answer Generation",
+        "Reasoning -> Temporal Reasoning",
+        "Reasoning -> Commonsense Reasoning",
+        "Answer Generation -> Contextual Question Answering",
+        "Answer Generation -> Abstractive"
     ],
-    "Definition": "In this task we ask you to write an implausible answer to a question that involves \u201cevent duration\", based on a given sentence. Here, event duration is defined as the understanding of how long events typically last. For example, \u201cbrushing teeth\u201d, usually takes few minutes.\nThings to avoid: -\nEmphasis & Caution: Even though there exist multiple wrong answers, we only need a single wrong answer. Please try to keep your \"answer\" as simple as possible. Concise and simple \"answer\" is preferred over those complex and verbose ones.  ",
+    "Definition": "In this task we ask you to write an implausible answer to a question that involves “event duration\", based on a given sentence. Here, event duration is defined as the understanding of how long events typically last. For example, “brushing teeth”, usually takes few minutes.\nThings to avoid: -\nEmphasis & Caution: Even though there exist multiple wrong answers, we only need a single wrong answer. Please try to keep your \"answer\" as simple as possible. Concise and simple \"answer\" is preferred over those complex and verbose ones.  ",
     "Positive Examples": [
         {
             "input": "Sentence: Jack played basketball after school, after which he was very tired. \nQuestion: How long did Jack play basketball?",
@@ -348,7 +352,7 @@
             ]
         },
         {
-            "input": "Sentence: Zarco and Teixeira were appointed co-go\u00advern\u00adors of Madeira, while Perestrelo was awarded Porto Santo. \nQuestion: How long were Zarco and Teixeira co-governors?",
+            "input": "Sentence: Zarco and Teixeira were appointed co-go­vern­ors of Madeira, while Perestrelo was awarded Porto Santo. \nQuestion: How long were Zarco and Teixeira co-governors?",
             "output": [
                 "36 years.",
                 "zarco and teixeira were co-governors for a term of 36 years.",
@@ -2665,7 +2669,7 @@
             ]
         },
         {
-            "input": "Sentence: Sarawak's coast and jungle interior were controlled by the Iban\u2002\u2014\u2002Sea Dayak pirates and Land Dayak slash-and-burn farmers. \nQuestion: How long did it take for the Iban to take control of Sarawak's coast?",
+            "input": "Sentence: Sarawak's coast and jungle interior were controlled by the Iban — Sea Dayak pirates and Land Dayak slash-and-burn farmers. \nQuestion: How long did it take for the Iban to take control of Sarawak's coast?",
             "output": [
                 "2 seconds.",
                 "a few seconds.",
@@ -3400,7 +3404,7 @@
             ]
         },
         {
-            "input": "Sentence: No, of course not\u2026 I\u2013  \"Then why,\" Arthur barreled on, \"do you disregard them all and return to this superstitious babble, this morbid, paganistic drivel?. \nQuestion: How long has Arthur been talking?",
+            "input": "Sentence: No, of course not… I–  \"Then why,\" Arthur barreled on, \"do you disregard them all and return to this superstitious babble, this morbid, paganistic drivel?. \nQuestion: How long has Arthur been talking?",
             "output": [
                 "two seconds.",
                 "10 centuries.",
@@ -3413,7 +3417,7 @@
             ]
         },
         {
-            "input": "Sentence: Britain seized Gibraltar\u2002\u2014\u2002in the name of the Hapsburg claimant\u2002\u2014\u2002and retained it when the war was over. \nQuestion: How long did the war happen?",
+            "input": "Sentence: Britain seized Gibraltar — in the name of the Hapsburg claimant — and retained it when the war was over. \nQuestion: How long did the war happen?",
             "output": [
                 "3 day.",
                 "1 century.",
@@ -5000,7 +5004,7 @@
             ]
         },
         {
-            "input": "Sentence: , followed by other ports at Hy\u00e8res, Antibes, and Nice. \nQuestion: How long were you at the ports?",
+            "input": "Sentence: , followed by other ports at Hyères, Antibes, and Nice. \nQuestion: How long were you at the ports?",
             "output": [
                 "four seconds.",
                 "i stayed at each port for 30 years each.",
@@ -5288,5 +5292,16 @@
     ],
     "Instruction_language": [
         "English"
+    ],
+    "Domains": [
+        "News",
+        "Wikipedia",
+        "Law",
+        "Justice",
+        "History",
+        "History -> 9/11 Reports",
+        "Anthropology",
+        "School Science Textbooks",
+        "Fiction"
     ]
-}
+}