feat: Eval meta

backend/app/api/endpoints/base/task.py

@@ -100,9 +100,9 @@ async def sign_in_consent(model: SignInConsentRequest):
     return TaskService().sign_in_consent(model.task_id, model.user_id)
 
 
-@router.post("/check_sign_consent", response_model={})
-async def check_sign_consent(model: CheckSignConsentRequest):
-    return TaskService().check_sign_consent(model.task_id, model.user_id)
+@router.post("/check_signed_consent", response_model={})
+async def check_signed_consent(model: CheckSignConsentRequest):
+    return TaskService().check_signed_consent(model.task_id, model.user_id)
 
 
 @router.post("/update_config_yaml", response_model={})

backend/app/domain/services/base/task.py

@@ -256,8 +256,8 @@ def get_active_tasks_by_user_id(self, user_id: int):
         active_tasks = [task for task in all_tasks if task["id"] in active_tasks]
         return active_tasks
 
-    def check_sign_consent(self, task_id: int, user_id: int):
-        if self.historical_task_repository.check_sign_consent(task_id, user_id):
+    def check_signed_consent(self, task_id: int, user_id: int):
+        if self.historical_task_repository.check_signed_consent(task_id, user_id):
             return True
         return False
 

backend/app/domain/services/builder_and_evaluation/builder.py

@@ -164,11 +164,12 @@ def create_task_definition(self, name_task: str, repo: str) -> str:
 
     def create_ecs_endpoint(self, name_task: str, repo: str) -> str:
         task_definition = self.create_task_definition(name_task, repo)
+        service_name = f"{name_task}-{randbelow(100000)}"
         launch_type = os.getenv("LAUNCH_TYPE")
         if launch_type == "FARGATE":
             run_service = self.ecs.create_service(
                 cluster=os.getenv("CLUSTER_TASK_EVALUATION"),
-                serviceName=name_task,
+                serviceName=service_name,
                 taskDefinition=task_definition,
                 desiredCount=1,
                 networkConfiguration={
@@ -186,7 +187,7 @@ def create_ecs_endpoint(self, name_task: str, repo: str) -> str:
         else:
             run_service = self.ecs.create_service(
                 cluster=os.getenv("CLUSTER_TASK_EVALUATION"),
-                serviceName=name_task,
+                serviceName=service_name,
                 taskDefinition=task_definition,
                 desiredCount=1,
                 launchType=launch_type,
@@ -204,7 +205,8 @@ def create_ecs_endpoint(self, name_task: str, repo: str) -> str:
             else:
                 arn_service = describe_service["services"][0]["serviceArn"]
                 run_task = self.ecs.list_tasks(
-                    cluster=os.getenv("CLUSTER_TASK_EVALUATION"), serviceName=name_task
+                    cluster=os.getenv("CLUSTER_TASK_EVALUATION"),
+                    serviceName=service_name,
                 )["taskArns"]
                 describe_task = self.ecs.describe_tasks(
                     cluster=os.getenv("CLUSTER_TASK_EVALUATION"), tasks=run_task

backend/app/domain/services/builder_and_evaluation/evaluation.py

@@ -355,14 +355,13 @@ def evaluation_with_selected_langs(
     ) -> dict:
         selected_langs = selected_langs.split(",")
         task_configuration = self.get_task_configuration(task_id)
+        folder_name = model_s3_zip.split("/")[-1].split(".")[0]
+        os.mkdir(f"./app/models/{folder_name}")
+        os.mkdir(f"./app/models/{folder_name}/datasets/")
         genders = task_configuration["submit_config"]["genders"]
         ip, model_name, folder_name, arn_service, repo_name = self.get_model_ip(
             model_s3_zip
         )
-        print("ciro_ip", ip)
-        folder_name = model_s3_zip.split("/")[-1].split(".")[0]
-        os.mkdir(f"./app/models/{folder_name}")
-        os.mkdir(f"./app/models/{folder_name}/datasets/")
         for lang in selected_langs:
             dataset = self.dataset_repository.get_scoring_datasets(task_id, lang)[0]
             dataset = self.download_dataset(task_code, dataset, folder_name)
@@ -414,8 +413,8 @@ def evaluation(
         model_s3_zip: str,
         model_id: int,
         user_id: int,
-        evaluate_no_scoring_datasets: bool = False,
         selected_langs: str = None,
+        evaluate_not_scoring_datasets: bool = False,
     ) -> dict:
         tasks = self.task_repository.get_model_id_and_task_code(task_code)
         if selected_langs is not None and len(selected_langs) > 0:
@@ -482,7 +481,7 @@ def evaluation(
                 selected_langs,
             )
             new_scores.append(new_score)
-        if evaluate_no_scoring_datasets:
+        if evaluate_not_scoring_datasets:
             jsonl_not_scoring_datasets = self.get_not_scoring_datasets(tasks.id)
             not_scoring_datasets = self.downloads_not_scoring_datasets(
                 jsonl_not_scoring_datasets,
@@ -506,8 +505,6 @@ def evaluation(
                 new_scores.append(new_score)
         self.builder.delete_repository(repo_name)
         self.logger.info("Create light model")
-        url_light_model = self.builder.create_light_model(model_name, folder_name)
-        self.model_repository.update_light_model(model_id, url_light_model)
         self.model_repository.update_model_status(model_id)
         self.logger.info("Clean folder and service")
         self.clean_folder_and_service(folder_name, arn_service)

backend/app/infrastructure/repositories/dataset.py

@@ -19,11 +19,9 @@ def get_scoring_datasets(self, task_id: int, dataset_name: str = None) -> dict:
             (self.model.access_type == "scoring") & (self.model.tid == task_id)
         )
         if dataset_name:
-            print("cirito, dataset_name:", dataset_name)
             scoring_datasets = scoring_datasets.filter(self.model.name == dataset_name)
         jsonl_scoring_datasets = []
         for scoring_dataset in scoring_datasets:
-            print("cirito")
             jsonl_scoring_dataset = {}
             jsonl_scoring_dataset["dataset"] = scoring_dataset.name
             jsonl_scoring_dataset["round_id"] = scoring_dataset.rid

backend/app/infrastructure/repositories/historical_data.py

@@ -45,7 +45,7 @@ def check_if_historical_data_exists(self, task_id: int, user_id: int, data: str)
             .first()
         )
 
-    def check_sign_consent(self, task_id: int, user_id: int):
+    def check_signed_consent(self, task_id: int, user_id: int):
         return (
             self.session.query(HistoricalData)
             .filter(HistoricalData.task_id == task_id)

backend/dynalab/base-model/requirements.txt

@@ -1,4 +1,4 @@
-fastapi==0.63.0
+fastapi==0.109.1
 mangum==0.10.0
 pydantic==1.7.3
 python-dotenv==0.15.0

backend/dynalab/example/base-model-nli/requirements.txt

@@ -1,4 +1,4 @@
-fastapi==0.63.0
+fastapi==0.109.1
 mangum==0.10.0
 pydantic==1.7.3
 python-dotenv==0.15.0

backend/dynalab/example/roberta-model-nli/requirements.txt

@@ -1,14 +1,14 @@
 certifi==2020.12.5
 click==7.1.2
-fastapi==0.63.0
+fastapi==0.109.1
 h11==0.11.0
 httptools==0.1.1
 mangum==0.10.0
 pydantic==1.7.3
 python-dotenv==0.15.0
 python-json-logger==2.0.1
 PyYAML==5.3.1
-starlette==0.13.6
+sse_starlette==1.6.1
 typing-extensions==3.7.4.3
 uvicorn==0.13.2
 uvloop==0.14.0

backend/requirements.txt

@@ -1,6 +1,6 @@
 uvicorn>=0.15.0,<0.16.0
 boto3==1.24.56
-fastapi==0.87.0
+fastapi==0.109.1
 sqlalchemy==1.4.25
 python-dotenv==0.19.0
 PyMySQL[rsa]==1.0.2

frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/ChatWithInstructions.tsx

@@ -0,0 +1,95 @@
+import React, { FC, useContext, useEffect, useState } from "react";
+import SignContract from "new_front/components/Modals/SignContract";
+import SimpleChatbot from "new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/SimpleChatbot";
+import useFetch from "use-http";
+import { ContextAnnotationFactoryType } from "new_front/types/createSamples/createSamples/annotationFactory";
+import { ContextConfigType } from "new_front/types/createSamples/createSamples/annotationContext";
+import UserContext from "containers/UserContext";
+import Modal from "react-bootstrap/Modal";
+import { ChatHistoryType } from "new_front/types/createSamples/createSamples/utils";
+import { CreateInterfaceContext } from "new_front/context/CreateInterface/Context";
+
+const ChatWithInstructions: FC<
+  ContextAnnotationFactoryType & ContextConfigType
+> = ({ taskId, generative_context, setIsGenerativeContext, contextId }) => {
+  const [signInConsent, setSignInConsent] = useState(false);
+  const { post, response } = useFetch();
+  const [chatHistory, setChatHistory] = useState<ChatHistoryType>({
+    user: [],
+    bot: [],
+  });
+  const [modelName, setModelName] = useState("");
+  const [provider, setProvider] = useState("");
+  const [artifactsInput, setArtifactsInput] = useState<any>(
+    generative_context.artifacts,
+  );
+  const [finishConversation, setFinishConversation] = useState(false);
+  const { updateModelInputs } = useContext(CreateInterfaceContext);
+  const { user } = useContext(UserContext);
+
+  const checkIfUserIsSignedInConsent = async () => {
+    const signConsent = await post("/task/check_signed_consent", {
+      user_id: user.id,
+      task_id: taskId,
+    });
+    if (response.ok) {
+      setSignInConsent(signConsent);
+    }
+  };
+
+  const handleSignInConsent = async () => {
+    await post("/task/sign_in_consent", {
+      user_id: user.id,
+      task_id: taskId,
+    });
+    setSignInConsent(true);
+    window.location.reload();
+  };
+
+  useEffect(() => {
+    checkIfUserIsSignedInConsent();
+  }, [signInConsent]);
+
+  return (
+    <>
+      {signInConsent ? (
+        <div className="grid grid-cols-3 gap-3 divide-x-2">
+          <div className="col-span-2">
+            <SimpleChatbot
+              instructions={"<p>Instructions</p>"}
+              chatHistory={chatHistory}
+              username={user.username}
+              modelName={modelName}
+              provider={provider}
+              setChatHistory={setChatHistory}
+              updateModelInputs={updateModelInputs}
+              setIsGenerativeContext={setIsGenerativeContext}
+            />
+          </div>
+          <div className="col-span-1">
+            <div className="p-4 bg-white">
+              <h2 className="mb-4 text-2xl font-bold">Instructions</h2>
+              <p className="text-gray-600">
+                You are playing the part of a 47-year-old female patient who is
+                concerned about having problems with her memory. She keeps
+                forgetting basic things around the house, like where she put her
+                keys or why she walked into a room. She has sticky notes all
+                over the house to help her remember everything. She first
+                started noticing about six months ago, but it has gotten worse,
+                to the point that some of her colleagues at work have also asked
+                about it. She is worried that it could be something to do with
+                dementia.
+              </p>
+            </div>
+          </div>
+        </div>
+      ) : (
+        <Modal show={true} onHide={() => setSignInConsent} size="lg">
+          <SignContract handleClose={handleSignInConsent} />
+        </Modal>
+      )}
+    </>
+  );
+};
+
+export default ChatWithInstructions;

frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/Chatbot.tsx

@@ -1,31 +1,14 @@
 import GeneralButton from "new_front/components/Buttons/GeneralButton";
 import BasicInput from "new_front/components/Inputs/BasicInput";
 import EvaluateText from "new_front/components/Inputs/EvaluateText";
-import React, { FC, useEffect, useState } from "react";
+import React, { FC, useState } from "react";
 import { PacmanLoader } from "react-spinners";
 import useFetch from "use-http";
 import Swal from "sweetalert2";
-import { ChatHistoryType } from "new_front/types/createSamples/createSamples/utils";
+import { ChatbotProps } from "new_front/types/createSamples/createSamples/utils";
 import { Avatar } from "components/Avatar/Avatar";
 import parse from "html-react-parser";
 
-type ChatbotProps = {
-  instructions: string;
-  chatHistory: ChatHistoryType;
-  username: string;
-  model_name: string;
-  provider: string;
-  num_of_samples_chatbot: number;
-  num_interactions_chatbot: number;
-  finishConversation: boolean;
-  optionsSlider?: string[];
-  setChatHistory: (chatHistory: ChatHistoryType) => void;
-  showOriginalInteractions: () => void;
-  setFinishConversation: (finishConversation: boolean) => void;
-  updateModelInputs: (modelInputs: any) => void;
-  setIsGenerativeContext: (isGenerativeContext: boolean) => void;
-};
-
 const Chatbot: FC<ChatbotProps> = ({
   instructions,
   chatHistory,

frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/DescribeImage.tsx

@@ -0,0 +1,59 @@
+import GeneralButton from "new_front/components/Buttons/GeneralButton";
+import BasicInput from "new_front/components/Inputs/BasicInput";
+import React, { useState } from "react";
+
+type ImageUploadProps = {
+  setImage: (image: string) => void;
+};
+
+const ImageUpload = ({ setImage }: ImageUploadProps) => {
+  const handleImageChange = (e: any) => {
+    const file = e.target.files[0];
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      setImage(reader.result as string);
+    };
+    reader.readAsDataURL(file);
+  };
+
+  return (
+    <div className="flex justify-center align-center">
+      <div className="flex flex-col items-center py-2 space-y-3 align-center">
+        <input type="file" accept="image/*" onChange={handleImageChange} />
+      </div>
+    </div>
+  );
+};
+
+const DescribeImage = () => {
+  const [image, setImage] = useState<string>(
+    "https://w7.pngwing.com/pngs/527/625/png-transparent-scalable-graphics-computer-icons-upload-uploading-cdr-angle-text-thumbnail.png",
+  );
+  const [description, setDescription] = useState<string>("");
+
+  return (
+    <div className="flex justify-center align-center">
+      <div className="flex flex-col items-center py-2 space-y-3 align-center">
+        <img
+          height={640}
+          width={640}
+          src={image}
+          alt="src"
+          className="rounded-lg cursor-pointer"
+        />
+        <ImageUpload setImage={setImage} />
+        <BasicInput
+          placeholder="Enter text here. Do not copy and paste"
+          onChange={(e) => setDescription(e.target.value)}
+        />
+        <GeneralButton
+          onClick={() => console.log(description)}
+          text="Send"
+          className="px-4 mt-[2px] font-semibold border-0 font-weight-bold light-gray-bg task-action-btn "
+        />
+      </div>
+    </div>
+  );
+};
+
+export default DescribeImage;

frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/EvaluateTextsGenerative.tsx

@@ -71,7 +71,7 @@ const EvaluateTextsGenerative: FC<
   };
 
   const checkIfUserIsSignedInConsent = async () => {
-    const signConsent = await post("/task/check_sign_consent", {
+    const signConsent = await post("/task/check_signed_consent", {
       user_id: user.id,
       task_id: taskId,
     });