Support offline datasets

formerbox/data/binarizer.py

@@ -3,7 +3,7 @@
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass, field
 from io import TextIOWrapper
-from typing import Any, Dict, List, Text, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Text, Tuple, Type, Union
 
 import torch
 from datasets import Dataset, DatasetDict, load_dataset
@@ -134,13 +134,26 @@ def encode(self, instance: Dict[Text, Any]) -> Dict[Text, Any]:
         raise NotImplementedError()
 
     def process_dataset(
-        self, filename: Text, script_path: Text, remove_columns: List[Text]
+        self,
+        filename: Text,
+        script_path: Text,
+        script_version: Optional[Text],
+        remove_columns: List[Text],
     ) -> Union[Dataset, DatasetDict]:
+        # check if packaged scripts are set correctly
+        if script_path in ["csv", "json", "text"]:
+            if script_version is not None:
+                logger.error(
+                    "Script %s is packaged into datasets library."
+                    " Make sure you do not set `script_version` argument.",
+                    script_path,
+                )
+
         dataset = load_dataset(
             path=script_path,
             data_files=[filename],
             split="train",
-            script_version="master",
+            script_version=script_version,
         )
 
         dataset = dataset.map(

formerbox/data/binarizer_default.py

@@ -97,6 +97,7 @@ def binarize(
         dataset = self.process_dataset(
             filename,
             script_path="text",
+            script_version=None,
             remove_columns=["text"],
         )
 

formerbox/data/binarizer_translation.py

@@ -84,14 +84,20 @@ def binarize(
         # process source dataset
         src_filename = f"{filename}.{self.params.src_lang}"
         src_dataset = self.process_dataset(
-            src_filename, script_path="text", remove_columns=["text"]
+            src_filename,
+            script_path="text",
+            script_version=None,
+            remove_columns=["text"],
         )
 
         if self.params.tgt_lang is not None:
             # process target dataset if present
             tgt_filename = f"{filename}.{self.params.tgt_lang}"
             tgt_dataset = self.process_dataset(
-                tgt_filename, script_path="text", remove_columns=["text"]
+                tgt_filename,
+                script_path="text",
+                script_version=None,
+                remove_columns=["text"],
             )
 
             logger.info("Processing source and target files")