Skip to content

Commit

Permalink
feat: Add new dynalab with HF integration (#235)
Browse files Browse the repository at this point in the history
* feat: add new dynalab

* feat: add new submit interface
  • Loading branch information
Ciroye authored Jan 18, 2024
1 parent 0247456 commit 5345034
Show file tree
Hide file tree
Showing 12 changed files with 602 additions and 58 deletions.
13 changes: 7 additions & 6 deletions backend/app/domain/services/builder_and_evaluation/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ def get_ip_ecs_task(self, model: str, logger: logging.Logger) -> str:
zip_name, model_name = self.download_zip(os.getenv("AWS_S3_BUCKET"), model)
folder_name = self.decompress_zip_to_folder(zip_name)
logger.info("Decompress model")
repo_name = "{}-{}".format(folder_name, randbelow(100000))
repo_name = f"{folder_name}-{randbelow(100000)}"
repo = self.create_repository(repo_name)
logger.info(f"Create repo: {repo}")
tag = "latest"
Expand Down Expand Up @@ -301,13 +301,14 @@ def create_light_model(self, model_name: str, folder_name: str):
)
digest = self.get_digest_repo(repo_name)
repo = repo + "@" + digest
lambda_function_name = "{}-{}".format(model_name, randbelow(100000))
lambda_function_name = f"{model_name}-{randbelow(100000)}"
self.light_model_deployment(lambda_function_name, repo)
self.create_permission_lambda_function(lambda_function_name)
return self.create_url_light_model(lambda_function_name)

def delete_repository(self, repo_name: str):
self.ecr.delete_repository(
repositoryName=repo_name,
force=True,
)
if repo_name:
self.ecr.delete_repository(
repositoryName=repo_name,
force=True,
)
18 changes: 18 additions & 0 deletions backend/app/domain/services/builder_and_evaluation/builder_hf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Copyright (c) MLCommons and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.


class BuilderHF:
def __init__(self):
pass

def get_ip_model(self):
"""
This should return an ip, model name, folder_name, arn_service
as None and repo_name as None
"""
pass

def download_model(self, url: str) -> None:
pass
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def evaluate(metric: str, formatted_predictions: list, formatted_labels: list) -
prediction_labels_dict,
target_labels_dict,
target_tags_dict,
) = format_data_for_evaluation(formatted_predictions, formatted_labels, tags=True)
) = format_data_for_evaluation(formatted_predictions, formatted_labels, tags=False)
perf, perf_dict = _compute_metric(metric, predictions, labels)
score_obj = {}
score_obj["perf"] = perf
Expand Down Expand Up @@ -109,7 +109,6 @@ def evaluate_delta_metrics(
grouped_fairs: list,
list_delta_metrics: list,
) -> list:

"""
Calculates the delta metric given a perturb prefix,
comparing original predictions with robust and fair
Expand Down
98 changes: 49 additions & 49 deletions backend/app/domain/services/builder_and_evaluation/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,8 @@ def evaluation(

def clean_folder_and_service(self, folder_name: str, arn_service: str):
print("arn_service", arn_service)
self.builder.delete_ecs_service(str(arn_service))
if arn_service:
self.builder.delete_ecs_service(str(arn_service))
shutil.rmtree(f"./app/models/{folder_name}")

def get_finals_scores(self, task_id: int, prediction_dict: dict, tags: bool):
Expand Down Expand Up @@ -462,54 +463,53 @@ def save_predictions_dataset(
arn_service: str,
current_round: int = 1,
):
try:
print("dataset", dataset)
(
prediction_dict,
dataset_id,
model_id,
minutes_time_prediction,
num_samples,
folder_name,
tags,
) = self.heavy_prediction(
dataset, tasks.task_code, ip, model_id, folder_name
)
self.logger.info("Calculate memory utilization")
memory = self.get_memory_utilization(model_name)
self.logger.info("Calculate throughput")
throughput = self.get_throughput(num_samples, minutes_time_prediction)
self.logger.info("Calculate score")
final_scores, main_metric, metric = self.get_finals_scores(
tasks.id, prediction_dict, tags
)
print("current_round", current_round)
round_info = self.round_repository.get_round_info_by_round_and_task(
tasks.id, current_round
)
print("final_scores", final_scores)
print("main_metric", main_metric)
print("metric", metric)
new_score = {
"perf": main_metric["perf"],
"pretty_perf": main_metric["pretty_perf"],
"fairness": final_scores["fairness"],
"robustness": final_scores["robustness"],
"mid": model_id,
"r_realid": round_info.id,
"did": dataset_id,
"memory_utilization": memory,
"examples_per_second": throughput,
}
final_score = new_score.copy()
metric_name = str(metric)
final_score[metric_name] = main_metric["perf"]
new_score["metadata_json"] = json.dumps(final_score)
self.score_repository.add(new_score)
self.logger.info("Save score")
return new_score
except Exception:
self.clean_folder_and_service(folder_name, arn_service)
# try:
print("dataset", dataset)
(
prediction_dict,
dataset_id,
model_id,
minutes_time_prediction,
num_samples,
folder_name,
tags,
) = self.heavy_prediction(dataset, tasks.task_code, ip, model_id, folder_name)
self.logger.info("Calculate memory utilization")
memory = self.get_memory_utilization(model_name)
self.logger.info("Calculate throughput")
throughput = self.get_throughput(num_samples, minutes_time_prediction)
self.logger.info("Calculate score")
final_scores, main_metric, metric = self.get_finals_scores(
tasks.id, prediction_dict, False
)
print("current_round", current_round)
round_info = self.round_repository.get_round_info_by_round_and_task(
tasks.id, current_round
)
print("final_scores", final_scores)
print("main_metric", main_metric)
print("metric", metric)
new_score = {
"perf": main_metric["perf"],
"pretty_perf": main_metric["pretty_perf"],
"fairness": final_scores["fairness"],
"robustness": final_scores["robustness"],
"mid": model_id,
"r_realid": round_info.id,
"did": dataset_id,
"memory_utilization": memory,
"examples_per_second": throughput,
}
final_score = new_score.copy()
metric_name = str(metric)
final_score[metric_name] = main_metric["perf"]
new_score["metadata_json"] = json.dumps(final_score)
self.score_repository.add(new_score)
self.logger.info("Save score")
return new_score

# except Exception:
# self.clean_folder_and_service(folder_name, arn_service)

def evaluate_dataperf_decentralized(self, dataperf_response: dict):
model_id = dataperf_response["model_id"]
Expand Down
36 changes: 36 additions & 0 deletions frontends/web/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions frontends/web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"react-content-loader": "^6.2.0",
"react-csv": "^2.2.2",
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-ga": "^3.1.2",
"react-grid-carousel": "^1.0.1",
"react-hook-form": "^7.34.2",
Expand Down
2 changes: 1 addition & 1 deletion frontends/web/src/containers/App.js
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import Register from "new_front/pages/Login/Register";
import ResetPassword from "./ResetPassword";
import ScrollToTop from "./ScrollToTop.js";
import SubmitInterface from "./SubmitInterface.js";
import SubmitModel from "./SubmitModel";
import SubmitModel from "new_front/pages/SubmitModel/SubmitModel";
import TaskModelLeaderboardPage from "./TaskModelLeaderboardPage.js";
import TaskOwnerPage from "./TaskOwnerPage";
import TaskPage from "new_front/pages/Task/TaskPage";
Expand Down
40 changes: 40 additions & 0 deletions frontends/web/src/new_front/components/Buttons/ProgressBar.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import React from "react";

const ProgressBar = ({
progress,
text,
}: {
progress: number;
text: string;
}) => {
return (
<div
className="center-loading"
style={{
width: "50%",
}}
>
<h3> Please do not close this window</h3>
<p>
The upload depends on the size of the model and your internet
connection, please be patient 😊
</p>
<div className="progress">
<div
className="progress-bar progress-bar-striped progress-bar-animated"
role="progressbar"
style={{ width: `${progress * 100}%` }}
></div>
</div>
<h6>
{progress > 0 && progress < 0.5
? text
: progress >= 0.5
? "We are processing your model"
: ""}
</h6>
</div>
);
};

export default ProgressBar;
70 changes: 70 additions & 0 deletions frontends/web/src/new_front/components/DragAndDrop/FileUpload.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import React, { ChangeEvent } from "react";
import { useDropzone } from "react-dropzone";

interface FileUploadProps {
fileName?: string | undefined;
setFileName: React.Dispatch<React.SetStateAction<string | undefined>>;
handleChange: (e: ChangeEvent<HTMLInputElement>) => void;
register: any; // Adjust the type of register according to your useForm usage
}

const FileUpload: React.FC<FileUploadProps> = ({
fileName,
setFileName,
handleChange,
register,
}) => {
const { getRootProps, getInputProps, isDragActive } = useDropzone({
accept: {
"application/pdf": [".zip"],
},
onDrop: (acceptedFiles) => {
if (acceptedFiles.length > 0) {
setFileName(acceptedFiles[0].name);
register("file", { value: acceptedFiles[0] });
}
},
maxFiles: 1,
});

return (
<div className="max-w-md mx-auto">
{fileName ? (
<div className="mb-4">
<div className="p-4 bg-white border rounded-md shadow-md">
<div className="flex items-center justify-between gap-3">
<div className="">{fileName}</div>
<button
className="pt-1 text-red-500 hover:text-red-700"
onClick={() => setFileName(undefined)}
>
Delete
</button>
</div>
</div>
<div className="mt-4">
<input
{...getInputProps()}
onChange={handleChange}
className="hidden"
/>
</div>
</div>
) : (
<div
{...getRootProps()}
className={`bg-white border border-dashed rounded-md p-8 text-center ${
isDragActive ? "border-blue-500" : ""
}`}
>
<input {...getInputProps()} onChange={handleChange} />
<p className="text-gray-600">
Drag & drop your zip model here, or click to select a zip model
</p>
</div>
)}
</div>
);
};

export default FileUpload;
Loading

0 comments on commit 5345034

Please sign in to comment.