EvaluationSetClient for deepset cloud to fetch evaluation sets and la…

haystack/document_stores/deepsetcloud.py

@@ -24,6 +24,7 @@ def __init__(
         api_endpoint: Optional[str] = None,
         similarity: str = "dot_product",
         return_embedding: bool = False,
+        evaluation_set_name: str = "default",
     ):
         """
         A DocumentStore facade enabling you to interact with the documents stored in Deepset Cloud.
@@ -65,6 +66,10 @@ def __init__(
                 f"{indexing_info['pending_file_count']} files are pending to be indexed. Indexing status: {indexing_info['status']}"
             )
 
+        self.evaluation_set_client = DeepsetCloud.get_evaluation_set_client(
+            api_key=api_key, api_endpoint=api_endpoint, workspace=workspace, evaluation_set_name=evaluation_set_name
+        )
+
         super().__init__()
 
     def get_all_documents(
@@ -441,16 +446,45 @@ def write_documents(
         """
         raise NotImplementedError("DeepsetCloudDocumentStore currently does not support writing documents.")
 
+    def get_evaluation_set_names(self) -> List[str]:
+        """
+        Returns a list of names for uploaded evaluation sets to deepset cloud.
+
+        :return: list of evaluation set names
+        """
+        return self.evaluation_set_client.get_evaluation_set_names()
+
     def get_all_labels(
         self,
-        index: Optional[str] = None,
+        evaluation_set_name: Optional[str] = None,
         filters: Optional[Dict[str, Union[Dict, List, str, int, float, bool]]] = None,
         headers: Optional[Dict[str, str]] = None,
     ) -> List[Label]:
-        raise NotImplementedError("DeepsetCloudDocumentStore currently does not support labels.")
+        """
+        Returns a list of labels for the given index name.
 
-    def get_label_count(self, index: Optional[str] = None, headers: Optional[Dict[str, str]] = None) -> int:
-        raise NotImplementedError("DeepsetCloudDocumentStore currently does not support labels.")
+        :param evaluation_set_name: Optional name of evaluation set for which labels should be searched.
+                      If None, the DocumentStore's default evaluation_set_name (self.evaluation_set_name) will be used.
+        :filters: Not supported.
+        :param headers: Not supported.
+
+        :return: list of Labels.
+        """
+        return self.evaluation_set_client.get_labels(evaluation_set_name=evaluation_set_name)
+
+    def get_label_count(
+        self, evaluation_set_name: Optional[str] = None, headers: Optional[Dict[str, str]] = None
+    ) -> int:
+        """
+        Counts the number of labels for the given index and returns the value.
+
+        :param evaluation_set_name: Optional evaluation set name for which the labels should be counted.
+                      If None, the DocumentStore's default index (self.index) will be used.
+        :param headers: Not supported.
+
+        :return: number of labels for the given index
+        """
+        raise self.evaluation_set_client.get_labels_count(evaluation_set_name=evaluation_set_name)
 
     def write_labels(
         self,

haystack/utils/deepsetcloud.py

@@ -5,6 +5,8 @@
 import time
 from typing import Any, Dict, Generator, List, Optional, Tuple, Union
 
+from haystack import Label, Document, Answer
+
 try:
     from typing import Literal
 except ImportError:
@@ -635,6 +637,115 @@ def _build_workspace_url(self, workspace: Optional[str] = None):
         return self.client.build_workspace_url(workspace)
 
 
+class EvaluationSetClient:
+    def __init__(
+        self, client: DeepsetCloudClient, workspace: Optional[str] = None, evaluation_set_name: Optional[str] = None
+    ):
+        """
+        A client to communicate with Deepset Cloud evaluation sets and labels.
+
+        :param client: Deepset Cloud client
+        :param workspace: workspace in Deepset Cloud
+        :param evaluation_set_name: name of the evaluation set
+
+        """
+        self.client = client
+        self.workspace = workspace
+        self.evaluation_set_name = evaluation_set_name
+
+    def get_labels(self, evaluation_set_name: str, workspace: Optional[str] = None) -> List[Label]:
+        """
+        Searches for labels for a given evaluation set in deepset cloud. Returns a list of all found labels.
+        If no labels were found, raises DeepsetCloudError.
+
+        :param evaluation_set_name: name of the evaluation set for which labels should be fetched
+        :param workspace: Optional workspace in Deepset Cloud
+                          If None, the EvaluationSetClient's default workspace (self.workspace) will be used.
+
+        :return: list of Label
+        """
+        try:
+            evaluation_set = self._get_evaluation_set(evaluation_set_name=evaluation_set_name, workspace=workspace)[0]
+        except IndexError:
+            raise DeepsetCloudError(f"No evaluation set found with the name {evaluation_set_name}")
+
+        labels = self._get_labels_from_evaluation_set(
+            workspace=workspace, evaluation_set_id=evaluation_set["evaluation_set_id"]
+        )
+
+        return [
+            Label(
+                query=label_dict["query"],
+                document=Document(content=label_dict["context"]),
+                is_correct_answer=True,
+                is_correct_document=True,
+                origin="user-feedback",
+                answer=Answer(label_dict["answer"]),
+                id=label_dict["label_id"],
+                no_answer=False if label_dict.get("answer", None) else True,
+                pipeline_id=None,
+                created_at=None,
+                updated_at=None,
+                meta=label_dict["meta"],
+                filters={},
+            )
+            for label_dict in labels
+        ]
+
+    def get_labels_count(self, evaluation_set_name: Optional[str] = None, workspace: Optional[str] = None) -> int:
+        """
+        Counts labels for a given evaluation set in deepset cloud.
+
+        :param evaluation_set_name: Optional index in Deepset Cloud
+                      If None, the EvaluationSetClient's default index (self.index) will be used.
+        :param workspace: Optional workspace in Deepset Cloud
+                          If None, the EvaluationSetClient's default workspace (self.workspace) will be used.
+
+        :return: Number of labels for the given (or defaulting) index
+        """
+        evaluation_set = self._get_evaluation_set(evaluation_set_name=evaluation_set_name, workspace=workspace)
+        return evaluation_set[0]["total_labels"]
+
+    def get_evaluation_set_names(self, workspace: Optional[str] = None):
+        """
+        Searches for all evaluation set names in the given workspace in Deepset Cloud.
+
+        :param workspace: Optional workspace in Deepset Cloud
+                          If None, the EvaluationSetClient's default workspace (self.workspace) will be used.
+
+        :return: list of Label
+        """
+        evaluation_sets_response = self._get_evaluation_set(evaluation_set_name=None, workspace=workspace)
+
+        return [eval_set["name"] for eval_set in evaluation_sets_response]
+
+    def _get_evaluation_set(self, evaluation_set_name: Optional[str], workspace: Optional[str] = None) -> List[dict]:
+        if not evaluation_set_name:
+            evaluation_set_name = self.evaluation_set_name
+
+        url = self._build_workspace_url(workspace=workspace)
+        evaluation_set_url = f"{url}/evaluation_sets"
+
+        for response in self.client.get_with_auto_paging(
+            url=evaluation_set_url, query_params={"name": evaluation_set_name}
+        ):
+            return response.json().get("data", [])
+
+    def _get_labels_from_evaluation_set(
+        self, workspace: Optional[str] = None, evaluation_set_id: Optional[str] = None
+    ) -> Generator[dict]:
+        url = f"{self._build_workspace_url(workspace=workspace)}/evaluation_sets/{evaluation_set_id}"
+        labels = self.client.get(url=url).json()
+
+        for label in labels:
+            yield label
+
+    def _build_workspace_url(self, workspace: Optional[str] = None):
+        if workspace is None:
+            workspace = self.workspace
+        return self.client.build_workspace_url(workspace)
+
+
 class DeepsetCloud:
     """
     A facade to communicate with Deepset Cloud.
@@ -683,3 +794,25 @@ def get_pipeline_client(
         """
         client = DeepsetCloudClient(api_key=api_key, api_endpoint=api_endpoint)
         return PipelineClient(client=client, workspace=workspace, pipeline_config_name=pipeline_config_name)
+
+    @classmethod
+    def get_evaluation_set_client(
+        cls,
+        api_key: Optional[str] = None,
+        api_endpoint: Optional[str] = None,
+        workspace: str = "default",
+        evaluation_set_name: str = "default",
+    ) -> EvaluationSetClient:
+        """
+        Creates a client to communicate with Deepset Cloud labels.
+
+        :param api_key: Secret value of the API key.
+                        If not specified, will be read from DEEPSET_CLOUD_API_KEY environment variable.
+        :param api_endpoint: The URL of the Deepset Cloud API.
+                             If not specified, will be read from DEEPSET_CLOUD_API_ENDPOINT environment variable.
+        :param workspace: workspace in Deepset Cloud
+        :param evaluation_set_name: name of the evaluation set in Deepset Cloud
+
+        """
+        client = DeepsetCloudClient(api_key=api_key, api_endpoint=api_endpoint)
+        return EvaluationSetClient(client=client, workspace=workspace, evaluation_set_name=evaluation_set_name)

test/test_document_store.py

@@ -1,3 +1,6 @@
+from typing import List
+from uuid import uuid4
+
 import numpy as np
 import pandas as pd
 import pytest
@@ -1633,6 +1636,151 @@ def test_DeepsetCloudDocumentStore_query(deepset_cloud_document_store):
     assert len(filtered_docs) < len(docs)
 
 
+@pytest.mark.parametrize(
+    "body, expected_count",
+    [
+        (
+            {
+                "data": [
+                    {
+                        "evaluation_set_id": uuid4(),
+                        "name": DC_TEST_INDEX,
+                        "created_at": "2022-03-22T13:40:27.535Z",
+                        "matched_labels": 2,
+                        "total_labels": 10,
+                    }
+                ],
+                "has_more": False,
+                "total": 1,
+            },
+            10,
+        ),
+        (
+            {
+                "data": [
+                    {
+                        "evaluation_set_id": uuid4(),
+                        "name": DC_TEST_INDEX,
+                        "created_at": "2022-03-22T13:40:27.535Z",
+                        "matched_labels": 0,
+                        "total_labels": 0,
+                    }
+                ],
+                "has_more": False,
+                "total": 1,
+            },
+            0,
+        ),
+    ],
+)
+@responses.activate
+def test_DeepsetCloudDocumentStore_count_of_labels_for_evaluation_set(
+    deepset_cloud_document_store, body: dict, expected_count: int
+):
+    if MOCK_DC:
+        responses.add(
+            method=responses.GET,
+            url=f"{DC_API_ENDPOINT}/workspaces/default/evaluation_sets",
+            status=200,
+            body=body,
+            query_params={"name": DC_TEST_INDEX},
+        )
+    else:
+        responses.add_passthru(DC_API_ENDPOINT)
+
+    count = deepset_cloud_document_store.get_labels_count()
+    assert count == expected_count
+
+
+@responses.activate
+def test_DeepsetCloudDocumentStore_lists_evaluation_set_names(deepset_cloud_document_store):
+    if MOCK_DC:
+        responses.add(
+            method=responses.GET,
+            url=f"{DC_API_ENDPOINT}/workspaces/default/evaluation_sets",
+            status=200,
+            body={
+                "data": [
+                    {
+                        "evaluation_set_id": uuid4(),
+                        "name": DC_TEST_INDEX,
+                        "created_at": "2022-03-22T13:40:27.535Z",
+                        "matched_labels": 2,
+                        "total_labels": 10,
+                    }
+                ],
+                "has_more": False,
+                "total": 1,
+            },
+        )
+    else:
+        responses.add_passthru(DC_API_ENDPOINT)
+
+    names = deepset_cloud_document_store.get_evaluation_set_names()
+    assert names == [DC_TEST_INDEX]
+
+
+@responses.activate
+def test_DeepsetCloudDocumentStore_fetches_lables_for_evaluation_set(deepset_cloud_document_store):
+    if MOCK_DC:
+        responses.add(
+            method=responses.GET,
+            url=f"{DC_API_ENDPOINT}/workspaces/default/evaluation_sets",
+            status=200,
+            body=[
+                {
+                    "label_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
+                    "query": "What is berlin?",
+                    "answer": "biggest city in germany",
+                    "answer_start": 0,
+                    "answer_end": 0,
+                    "meta": {},
+                    "context": "Berlin is the biggest city in germany.",
+                    "external_file_name": "string",
+                    "file_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
+                    "state": "Label matching status",
+                    "candidates": "Candidates that were found in the label <-> file matching",
+                }
+            ],
+        )
+    else:
+        responses.add_passthru(DC_API_ENDPOINT)
+
+    labels = deepset_cloud_document_store.get_all_labels(evaluation_set_name=DC_TEST_INDEX)
+    assert labels == [
+        Label(
+            query="What is berlin?",
+            document=Document(content="Berlin is the biggest city in germany."),
+            is_correct_answer=True,
+            is_correct_document=True,
+            origin="user-feedback",
+            answer=Answer("biggest city in germany"),
+            id="3fa85f64-5717-4562-b3fc-2c963f66afa6",
+            no_answer=False,
+            pipeline_id=None,
+            created_at=None,
+            updated_at=None,
+            meta={},
+            filters={},
+        )
+    ]
+
+
+@responses.activate
+def test_DeepsetCloudDocumentStore_fetches_lables_for_evaluation_set_raises_deepsetclouderror_when_nothing_found(
+    deepset_cloud_document_store,
+):
+    if MOCK_DC:
+        responses.add(
+            method=responses.GET, url=f"{DC_API_ENDPOINT}/workspaces/default/evaluation_sets", status=200, body=[]
+        )
+    else:
+        responses.add_passthru(DC_API_ENDPOINT)
+
+    with pytest.raises(DeepsetCloudError):
+        deepset_cloud_document_store.get_all_labels(evaluation_set_name=DC_TEST_INDEX)
+
+
 @responses.activate
 def test_DeepsetCloudDocumentStore_query_by_embedding(deepset_cloud_document_store):
     query_emb = np.random.randn(768)