Data V2

allennlp/commands/evaluate.py

@@ -62,7 +62,7 @@
 from allennlp.commands.subcommand import Subcommand
 from allennlp.common.util import dump_metrics, prepare_environment
 from allennlp.data.dataset_readers.dataset_reader import DatasetReader
-from allennlp.data.iterators import DataIterator
+from allennlp.data import DataLoader
 from allennlp.models.archival import load_archive
 from allennlp.training.util import evaluate
 
@@ -173,15 +173,15 @@ def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
         model.vocab.extend_from_instances(instances=instances)
         model.extend_embedder_vocab(embedding_sources)
 
-    iterator_params = config.pop("validation_iterator", None)
-    if iterator_params is None:
-        iterator_params = config.pop("iterator")
+    instances.index_with(model.vocab)
+    data_loader_params = config.pop("validation_data_loader", None)
+    if data_loader_params is None:
+        data_loader_params = config.pop("data_loader")
     if args.batch_size:
-        iterator_params["batch_size"] = args.batch_size
-    iterator = DataIterator.from_params(iterator_params)
-    iterator.index_with(model.vocab)
+        data_loader_params["batch_size"] = args.batch_size
+    data_loader = DataLoader.from_params(dataset=instances, params=data_loader_params)
 
-    metrics = evaluate(model, instances, iterator, args.cuda_device, args.batch_weight_key)
+    metrics = evaluate(model, data_loader, args.cuda_device, args.batch_weight_key)
 
     logger.info("Finished evaluating.")
 

allennlp/commands/find_learning_rate.py

@@ -49,14 +49,16 @@
 import os
 import re
 from typing import List, Tuple
+import itertools
 
 from overrides import overrides
 
 from allennlp.commands.subcommand import Subcommand
 from allennlp.common import Params, Tqdm
 from allennlp.common.checks import check_for_gpu, ConfigurationError
 from allennlp.common.util import prepare_environment
-from allennlp.data import DataIterator, Vocabulary
+from allennlp.data import Vocabulary
+from allennlp.data import DataLoader
 from allennlp.models import Model
 from allennlp.training import Trainer, TrainerBase
 from allennlp.training.util import create_serialization_dir, datasets_from_params
@@ -211,11 +213,10 @@ def find_learning_rate_model(
         ),
     )
 
-    model = Model.from_params(vocab=vocab, params=params.pop("model"))
-    iterator = DataIterator.from_params(params.pop("iterator"))
-    iterator.index_with(vocab)
-
     train_data = all_datasets["train"]
+    train_data.index_with(vocab)
+    model = Model.from_params(vocab=vocab, params=params.pop("model"))
+    data_loader = DataLoader.from_params(dataset=train_data, params=params.pop("data_loader"))
 
     trainer_params = params.pop("trainer")
 
@@ -230,11 +231,8 @@ def find_learning_rate_model(
     trainer: Trainer = TrainerBase.from_params(  # type: ignore
         model=model,
         serialization_dir=serialization_dir,
-        iterator=iterator,
-        train_data=train_data,
-        validation_data=None,
+        data_loader=data_loader,
         params=trainer_params,
-        validation_iterator=None,
     )
 
     logger.info(
@@ -292,8 +290,8 @@ def search_learning_rate(
 
     trainer.model.train()
 
-    train_generator = trainer.iterator(trainer.train_data, shuffle=trainer.shuffle)
-    train_generator_tqdm = Tqdm.tqdm(train_generator, total=num_batches)
+    infinite_generator = itertools.cycle(trainer.data_loader)
+    train_generator_tqdm = Tqdm.tqdm(infinite_generator, total=num_batches)
 
     learning_rates = []
     losses = []

allennlp/commands/train.py

@@ -41,7 +41,7 @@
 import argparse
 import logging
 import os
-from typing import Any, Dict, Iterable, List, Optional
+from typing import Any, Dict, List, Optional
 
 import torch
 import torch.distributed as dist
@@ -53,7 +53,8 @@
 from allennlp.common.checks import check_for_gpu, ConfigurationError
 from allennlp.common import util as common_util
 from allennlp.common.plugins import import_plugins
-from allennlp.data import DataIterator, DatasetReader, Instance, Vocabulary
+from allennlp.data import DatasetReader, Vocabulary
+from allennlp.data import DataLoader
 from allennlp.models.archival import archive_model, CONFIG_NAME
 from allennlp.models.model import _DEFAULT_WEIGHTS, Model
 from allennlp.training.trainer_base import TrainerBase
@@ -296,7 +297,7 @@ def train_model(
         )
 
         # Creating `Vocabulary` objects from workers could be problematic since
-        # the data iterators in each worker will yield only `rank` specific
+        # the data loaders in each worker will yield only `rank` specific
         # instances. Hence it is safe to construct the vocabulary and write it
         # to disk before initializing the distributed context. The workers will
         # load the vocabulary from the path specified.
@@ -504,36 +505,33 @@ def __init__(
         serialization_dir: str,
         model: Model,
         trainer: TrainerBase,
-        evaluation_dataset: Iterable[Instance] = None,
-        evaluation_iterator: DataIterator = None,
+        evaluation_data_loader: DataLoader = None,
         evaluate_on_test: bool = False,
         batch_weight_key: str = "",
     ) -> None:
         self.serialization_dir = serialization_dir
         self.model = model
         self.trainer = trainer
-        self.evaluation_dataset = evaluation_dataset
-        self.evaluation_iterator = evaluation_iterator
+        self.evaluation_data_loader = evaluation_data_loader
         self.evaluate_on_test = evaluate_on_test
         self.batch_weight_key = batch_weight_key
 
     def run(self) -> Dict[str, Any]:
         return self.trainer.train()
 
     def finish(self, metrics: Dict[str, Any]):
-        if self.evaluation_dataset and self.evaluate_on_test:
+        if self.evaluation_data_loader and self.evaluate_on_test:
             logger.info("The model will be evaluated using the best epoch weights.")
             test_metrics = training_util.evaluate(
                 self.model,
-                self.evaluation_dataset,
-                self.evaluation_iterator,
+                self.evaluation_data_loader,
                 cuda_device=self.trainer.cuda_device,
                 batch_weight_key=self.batch_weight_key,
             )
 
             for key, value in test_metrics.items():
                 metrics["test_" + key] = value
-        elif self.evaluation_dataset:
+        elif self.evaluation_data_loader:
             logger.info(
                 "To evaluate on the test set after training, pass the "
                 "'evaluate_on_test' flag, or use the 'allennlp evaluate' command."
@@ -551,13 +549,13 @@ def from_partial_objects(
         dataset_reader: DatasetReader,
         train_data_path: str,
         model: Lazy[Model],
-        iterator: DataIterator,
+        data_loader: Lazy[DataLoader],
         trainer: Lazy[TrainerBase],
         vocabulary: Lazy[Vocabulary] = None,
         datasets_for_vocab_creation: List[str] = None,
         validation_dataset_reader: DatasetReader = None,
         validation_data_path: str = None,
-        validation_iterator: DataIterator = None,
+        validation_data_loader: Lazy[DataLoader] = None,
         test_data_path: str = None,
         evaluate_on_test: bool = False,
     ) -> "TrainModel":
@@ -595,9 +593,9 @@ def from_partial_objects(
         model: `Lazy[Model]`
             The model that we will train.  This is lazy because it depends on the `Vocabulary`;
             after constructing the vocabulary we call `model.construct(vocab=vocabulary)`.
-        iterator: `DataIterator`
-            The iterator we use to batch instances from the dataset reader at training and (by
-            default) validation time.
+        data_loader: `Lazy[DataLoader]`
+            The data_loader we use to batch instances from the dataset reader at training and (by
+            default) validation time. This is lazy because it takes a dataset in it's constructor.
         trainer: `Lazy[TrainerBase]`
             The `Trainer` that actually implements the training loop.  This is a lazy object because
             it depends on the model that's going to be trained.
@@ -614,9 +612,9 @@ def from_partial_objects(
             `dataset_reader`.
         validation_data_path: `str`, optional (default=None)
             If given, we will use this data for computing validation metrics and early stopping.
-        validation_iterator: `DataIterator`, optional (default=None)
-            If given, we will use this iterator for batching and scheduling instances for the
-            validation data, instead of `iterator`.
+        validation_data_loader: `Lazy[DataLoader]`, optional (default=None)
+            If given, the data_loader we use to batch instances from the dataset reader at
+            validation and test time. This is lazy because it takes a dataset in it's constructor.
         test_data_path: `str`, optional (default=None)
             If given, we will use this as test data.  This makes it available for vocab creation by
             default, but nothing else.
@@ -658,27 +656,42 @@ def from_partial_objects(
             vocabulary_path = os.path.join(serialization_dir, "vocabulary")
             vocabulary_.save_to_files(vocabulary_path)
 
-        iterator.index_with(model_.vocab)
-        validation_iterator = validation_iterator or iterator
-        validation_iterator.index_with(model_.vocab)  # it is ok to call this twice
+        for dataset in datasets.values():
+            dataset.index_with(model_.vocab)
+
+        data_loader_ = data_loader.construct(dataset=datasets["train"])
+        validation_data = datasets.get("validation")
+        if validation_data is not None:
+            # Because of the way Lazy[T] works, we can't check it's existence
+            # _before_ we've tried to construct it. It returns None if it is not
+            # present, so we try to construct it first, and then afterward back off
+            # to the data_loader configuration used for training if it returns None.
+            validation_data_loader_ = validation_data_loader.construct(dataset=validation_data)
+            if validation_data_loader_ is None:
+                validation_data_loader_ = data_loader.construct(dataset=validation_data)
+        else:
+            validation_data_loader_ = None
+
+        test_data = datasets.get("test")
+        if test_data is not None:
+            test_data_loader = validation_data_loader.construct(dataset=test_data)
+            if test_data_loader is None:
+                test_data_loader = data_loader.construct(dataset=test_data)
+        else:
+            test_data_loader = None
 
         # We don't need to pass serialization_dir and local_rank here, because they will have been
         # passed through the trainer by from_params already, because they were keyword arguments to
         # construct this class in the first place.
         trainer_ = trainer.construct(
-            model=model_,
-            iterator=iterator,
-            train_data=datasets["train"],
-            validation_iterator=validation_iterator,
-            validation_data=datasets.get("validation"),
+            model=model_, data_loader=data_loader_, validation_data_loader=validation_data_loader_,
         )
 
         return cls(
             serialization_dir=serialization_dir,
             model=model_,
             trainer=trainer_,
-            evaluation_dataset=datasets.get("test"),
-            evaluation_iterator=validation_iterator,
+            evaluation_data_loader=test_data_loader,
             evaluate_on_test=evaluate_on_test,
             batch_weight_key=batch_weight_key,
         )

allennlp/common/lazy.py

@@ -1,4 +1,4 @@
-from typing import Callable, Generic, TypeVar
+from typing import Callable, Generic, TypeVar, Optional
 
 T = TypeVar("T")
 
@@ -20,10 +20,27 @@ class Lazy(Generic[T]):
     The actual implementation here is incredibly simple; the logic that handles the lazy
     construction is actually found in `FromParams`, where we have a special case for a `Lazy` type
     annotation.
+
+    !!! Warning
+        The way this class is used in from_params means that optional constructor arguments CANNOT
+        be compared to `None` _before_ it is constructed. See the example below for correct usage.
+
+    ```
+    @classmethod
+    def my_constructor(cls, some_object: Lazy[MyObject] = None) -> MyClass:
+        ...
+        # WRONG! some_object will never be None at this point, it will be
+        # a Lazy[] that returns None
+        obj = some_object or MyObjectDefault()
+        # CORRECT:
+        obj = some_object.construct(kwarg=kwarg) or MyObjectDefault()
+        ...
+    ```
+
     """
 
     def __init__(self, constructor: Callable[..., T]):
         self._constructor = constructor
 
-    def construct(self, **kwargs) -> T:
+    def construct(self, **kwargs) -> Optional[T]:
         return self._constructor(**kwargs)

allennlp/common/testing/model_test_case.py

@@ -7,7 +7,8 @@
 from allennlp.commands.train import train_model_from_file
 from allennlp.common import Params
 from allennlp.common.testing.test_case import AllenNlpTestCase
-from allennlp.data import DataIterator, DatasetReader, Vocabulary
+from allennlp.data import DatasetReader, Vocabulary
+from allennlp.data import DataLoader
 from allennlp.data.batch import Batch
 from allennlp.models import load_archive, Model
 
@@ -25,7 +26,7 @@ def set_up_model(self, param_file, dataset_file):
 
         reader = DatasetReader.from_params(params["dataset_reader"])
         # The dataset reader might be lazy, but a lazy list here breaks some of our tests.
-        instances = list(reader.read(str(dataset_file)))
+        instances = reader.read(str(dataset_file))
         # Use parameters for vocabulary if they are present in the config file, so that choices like
         # "non_padded_namespaces", "min_count" etc. can be set if needed.
         if "vocabulary" in params:
@@ -35,11 +36,12 @@ def set_up_model(self, param_file, dataset_file):
             vocab = Vocabulary.from_instances(instances)
         self.vocab = vocab
         self.instances = instances
+        self.instances.index_with(vocab)
         self.model = Model.from_params(vocab=self.vocab, params=params["model"])
 
         # TODO(joelgrus) get rid of these
         # (a lot of the model tests use them, so they'll have to be changed)
-        self.dataset = Batch(self.instances)
+        self.dataset = Batch(list(self.instances))
         self.dataset.index_instances(self.vocab)
 
     def ensure_model_can_train_save_and_load(
@@ -93,24 +95,27 @@ def ensure_model_can_train_save_and_load(
         params = Params.from_file(param_file, params_overrides=overrides)
         reader = DatasetReader.from_params(params["dataset_reader"])
 
-        # Need to duplicate params because Iterator.from_params will consume.
-        iterator_params = params["iterator"]
-        iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict()))
-
-        iterator = DataIterator.from_params(iterator_params)
-        iterator2 = DataIterator.from_params(iterator_params2)
-
-        # We'll check that even if we index the dataset with each model separately, we still get
-        # the same result out.
         print("Reading with original model")
         model_dataset = reader.read(params["validation_data_path"])
-        iterator.index_with(model.vocab)
-        model_batch = next(iterator(model_dataset, shuffle=False))
+        model_dataset.index_with(model.vocab)
 
         print("Reading with loaded model")
         loaded_dataset = reader.read(params["validation_data_path"])
-        iterator2.index_with(loaded_model.vocab)
-        loaded_batch = next(iterator2(loaded_dataset, shuffle=False))
+        loaded_dataset.index_with(loaded_model.vocab)
+
+        # Need to duplicate params because DataLoader.from_params will consume.
+        data_loader_params = params["data_loader"]
+        data_loader_params["shuffle"] = False
+        data_loader_params2 = Params(copy.deepcopy(data_loader_params.as_dict()))
+
+        data_loader = DataLoader.from_params(dataset=model_dataset, params=data_loader_params)
+        data_loader2 = DataLoader.from_params(dataset=loaded_dataset, params=data_loader_params2)
+
+        # We'll check that even if we index the dataset with each model separately, we still get
+        # the same result out.
+        model_batch = next(iter(data_loader))
+
+        loaded_batch = next(iter(data_loader2))
 
         # Check gradients are None for non-trainable parameters and check that
         # trainable parameters receive some gradient if they are trainable.

allennlp/data/__init__.py

@@ -1,3 +1,4 @@
+from allennlp.data.dataloader import DataLoader, allennlp_collate
 from allennlp.data.dataset_readers.dataset_reader import DatasetReader
 from allennlp.data.fields.field import DataArray, Field
 from allennlp.data.fields.text_field import TextFieldTensors