EvaluationSetClient for deepset cloud to fetch evaluation sets and la…

docs/_src/api/api/document_store.md

@@ -4257,6 +4257,61 @@ exists.
 
 None
 
+<a id="deepsetcloud.DeepsetCloudDocumentStore.get_evaluation_sets"></a>
+
+#### get\_evaluation\_sets
+
+```python
+def get_evaluation_sets() -> List[dict]
+```
+
+Returns a list of uploaded evaluation sets to deepset cloud.
+
+**Returns**:
+
+list of evaluation sets as dicts
+These contain ("name", "evaluation_set_id", "created_at", "matched_labels", "total_labels") as fields.
+
+<a id="deepsetcloud.DeepsetCloudDocumentStore.get_all_labels"></a>
+
+#### get\_all\_labels
+
+```python
+def get_all_labels(index: Optional[str] = None, filters: Optional[Dict[str, Union[Dict, List, str, int, float, bool]]] = None, headers: Optional[Dict[str, str]] = None) -> List[Label]
+```
+
+Returns a list of labels for the given index name.
+
+**Arguments**:
+
+- `label_index`: Optional name of evaluation set for which labels should be searched.
+If None, the DocumentStore's default label_index (self.label_index) will be used.
+- `headers`: Not supported.
+
+**Returns**:
+
+list of Labels.
+
+<a id="deepsetcloud.DeepsetCloudDocumentStore.get_label_count"></a>
+
+#### get\_label\_count
+
+```python
+def get_label_count(label_index: Optional[str] = None, headers: Optional[Dict[str, str]] = None) -> int
+```
+
+Counts the number of labels for the given index and returns the value.
+
+**Arguments**:
+
+- `label_index`: Optional evaluation set name for which the labels should be counted.
+If None, the DocumentStore's default index (self.index) will be used.
+- `headers`: Not supported.
+
+**Returns**:
+
+number of labels for the given index
+
 <a id="pinecone"></a>
 
 # Module pinecone

haystack/document_stores/deepsetcloud.py

@@ -65,6 +65,10 @@ def __init__(
                 f"{indexing_info['pending_file_count']} files are pending to be indexed. Indexing status: {indexing_info['status']}"
             )
 
+        self.evaluation_set_client = DeepsetCloud.get_evaluation_set_client(
+            api_key=api_key, api_endpoint=api_endpoint, workspace=workspace, label_index=index
+        )
+
         super().__init__()
 
     def get_all_documents(
@@ -452,16 +456,44 @@ def write_documents(
         """
         raise NotImplementedError("DeepsetCloudDocumentStore currently does not support writing documents.")
 
+    def get_evaluation_sets(self) -> List[dict]:
+        """
+        Returns a list of uploaded evaluation sets to deepset cloud.
+
+        :return: list of evaluation sets as dicts
+                 These contain ("name", "evaluation_set_id", "created_at", "matched_labels", "total_labels") as fields.
+        """
+        return self.evaluation_set_client.get_evaluation_sets()
+
     def get_all_labels(
         self,
         index: Optional[str] = None,
         filters: Optional[Dict[str, Union[Dict, List, str, int, float, bool]]] = None,
         headers: Optional[Dict[str, str]] = None,
     ) -> List[Label]:
-        raise NotImplementedError("DeepsetCloudDocumentStore currently does not support labels.")
+        """
+        Returns a list of labels for the given index name.
 
-    def get_label_count(self, index: Optional[str] = None, headers: Optional[Dict[str, str]] = None) -> int:
-        raise NotImplementedError("DeepsetCloudDocumentStore currently does not support labels.")
+        :param label_index: Optional name of evaluation set for which labels should be searched.
+                      If None, the DocumentStore's default label_index (self.label_index) will be used.
+        :filters: Not supported.
+        :param headers: Not supported.
+
+        :return: list of Labels.
+        """
+        return self.evaluation_set_client.get_labels(label_index=index)
+
+    def get_label_count(self, label_index: Optional[str] = None, headers: Optional[Dict[str, str]] = None) -> int:
+        """
+        Counts the number of labels for the given index and returns the value.
+
+        :param label_index: Optional evaluation set name for which the labels should be counted.
+                      If None, the DocumentStore's default index (self.index) will be used.
+        :param headers: Not supported.
+
+        :return: number of labels for the given index
+        """
+        return self.evaluation_set_client.get_labels_count(label_index=label_index)
 
     def write_labels(
         self,

haystack/utils/deepsetcloud.py

@@ -5,6 +5,8 @@
 import time
 from typing import Any, Dict, Generator, List, Optional, Tuple, Union
 
+from haystack.schema import Label, Document, Answer
+
 try:
     from typing import Literal
 except ImportError:
@@ -637,6 +639,116 @@ def _build_workspace_url(self, workspace: Optional[str] = None):
         return self.client.build_workspace_url(workspace)
 
 
+class EvaluationSetClient:
+    def __init__(self, client: DeepsetCloudClient, workspace: Optional[str] = None, label_index: Optional[str] = None):
+        """
+        A client to communicate with Deepset Cloud evaluation sets and labels.
+
+        :param client: Deepset Cloud client
+        :param workspace: workspace in Deepset Cloud
+        :param label_index: name of the label index
+
+        """
+        self.client = client
+        self.workspace = workspace
+        self.label_index = label_index
+
+    def get_labels(self, label_index: Optional[str], workspace: Optional[str] = None) -> List[Label]:
+        """
+        Searches for labels for a given evaluation set in deepset cloud. Returns a list of all found labels.
+        If no labels were found, raises DeepsetCloudError.
+
+        :param label_index: name of the evaluation set for which labels should be fetched
+        :param workspace: Optional workspace in Deepset Cloud
+                          If None, the EvaluationSetClient's default workspace (self.workspace) will be used.
+
+        :return: list of Label
+        """
+        try:
+            evaluation_set = next(self._get_evaluation_set(label_index=label_index, workspace=workspace))
+        except StopIteration:
+            raise DeepsetCloudError(f"No evaluation set found with the name {label_index}")
+
+        labels = self._get_labels_from_evaluation_set(
+            workspace=workspace, evaluation_set_id=evaluation_set["evaluation_set_id"]
+        )
+
+        return [
+            Label(
+                query=label_dict["query"],
+                document=Document(content=label_dict["context"]),
+                is_correct_answer=True,
+                is_correct_document=True,
+                origin="user-feedback",
+                answer=Answer(label_dict["answer"]),
+                id=label_dict["label_id"],
+                no_answer=False if label_dict.get("answer", None) else True,
+                pipeline_id=None,
+                created_at=None,
+                updated_at=None,
+                meta=label_dict["meta"],
+                filters={},
+            )
+            for label_dict in labels
+        ]
+
+    def get_labels_count(self, label_index: Optional[str] = None, workspace: Optional[str] = None) -> int:
+        """
+        Counts labels for a given evaluation set in deepset cloud.
+
+        :param label_index: Optional index in Deepset Cloud
+                            If None, the EvaluationSetClient's default index (self.index) will be used.
+        :param workspace: Optional workspace in Deepset Cloud
+                          If None, the EvaluationSetClient's default workspace (self.workspace) will be used.
+
+        :return: Number of labels for the given (or defaulting) index
+        """
+        try:
+            evaluation_set = next(self._get_evaluation_set(label_index=label_index, workspace=workspace))
+        except StopIteration:
+            raise DeepsetCloudError(f"No evaluation set found with the name {label_index}")
+
+        return evaluation_set["total_labels"]
+
+    def get_evaluation_sets(self, workspace: Optional[str] = None) -> List[dict]:
+        """
+        Searches for all evaluation set names in the given workspace in Deepset Cloud.
+
+        :param workspace: Optional workspace in Deepset Cloud
+                          If None, the EvaluationSetClient's default workspace (self.workspace) will be used.
+
+        :return: List of dictionaries that represent deepset Cloud evaluation sets.
+                 These contain ("name", "evaluation_set_id", "created_at", "matched_labels", "total_labels") as fields.
+        """
+        evaluation_sets_response = self._get_evaluation_set(label_index=None, workspace=workspace)
+
+        return [eval_set for eval_set in evaluation_sets_response]
+
+    def _get_evaluation_set(self, label_index: Optional[str], workspace: Optional[str] = None) -> Generator:
+        if not label_index:
+            label_index = self.label_index
+
+        url = self._build_workspace_url(workspace=workspace)
+        evaluation_set_url = f"{url}/evaluation_sets"
+
+        for response in self.client.get_with_auto_paging(url=evaluation_set_url, query_params={"name": label_index}):
+            yield response
+
+    def _get_labels_from_evaluation_set(
+        self, workspace: Optional[str] = None, evaluation_set_id: Optional[str] = None
+    ) -> Generator:
+        url = f"{self._build_workspace_url(workspace=workspace)}/evaluation_sets/{evaluation_set_id}"
+        labels = self.client.get(url=url).json()
+
+        for label in labels:
+            yield label
+
+    def _build_workspace_url(self, workspace: Optional[str] = None):
+        if workspace is None:
+            workspace = self.workspace
+        return self.client.build_workspace_url(workspace)
+
+
 class DeepsetCloud:
     """
     A facade to communicate with Deepset Cloud.
@@ -685,3 +797,25 @@ def get_pipeline_client(
         """
         client = DeepsetCloudClient(api_key=api_key, api_endpoint=api_endpoint)
         return PipelineClient(client=client, workspace=workspace, pipeline_config_name=pipeline_config_name)
+
+    @classmethod
+    def get_evaluation_set_client(
+        cls,
+        api_key: Optional[str] = None,
+        api_endpoint: Optional[str] = None,
+        workspace: str = "default",
+        label_index: str = "default",
+    ) -> EvaluationSetClient:
+        """
+        Creates a client to communicate with Deepset Cloud labels.
+
+        :param api_key: Secret value of the API key.
+                        If not specified, will be read from DEEPSET_CLOUD_API_KEY environment variable.
+        :param api_endpoint: The URL of the Deepset Cloud API.
+                             If not specified, will be read from DEEPSET_CLOUD_API_ENDPOINT environment variable.
+        :param workspace: workspace in Deepset Cloud
+        :param label_index: name of the evaluation set in Deepset Cloud
+
+        """
+        client = DeepsetCloudClient(api_key=api_key, api_endpoint=api_endpoint)
+        return EvaluationSetClient(client=client, workspace=workspace, label_index=label_index)