[refactoring] split queue.py in 3 modules (#2930)

* extract lock

* move metrics

* move queue

* new_queue -> queue

* fix

* fix

jobs/cache_maintenance/src/cache_maintenance/queue_metrics.py

@@ -3,7 +3,8 @@
 
 import logging
 
-from libcommon.queue import JobTotalMetricDocument, Queue, WorkerSizeJobsCountDocument
+from libcommon.queue.jobs import Queue
+from libcommon.queue.metrics import JobTotalMetricDocument, WorkerSizeJobsCountDocument
 
 
 def collect_queue_metrics() -> None:

jobs/cache_maintenance/tests/conftest.py

@@ -3,7 +3,7 @@
 
 from collections.abc import Iterator
 
-from libcommon.queue import _clean_queue_database
+from libcommon.queue.utils import _clean_queue_database
 from libcommon.resources import CacheMongoResource, QueueMongoResource
 from libcommon.simple_cache import _clean_cache_database
 from pytest import MonkeyPatch, fixture

jobs/cache_maintenance/tests/test_collect_queue_metrics.py

@@ -4,13 +4,8 @@
 from unittest.mock import patch
 
 import pytest
-from libcommon.queue import (
-    JobsCountByWorkerSize,
-    JobsTotalByTypeAndStatus,
-    JobTotalMetricDocument,
-    Queue,
-    WorkerSizeJobsCountDocument,
-)
+from libcommon.queue.jobs import JobsCountByWorkerSize, JobsTotalByTypeAndStatus, Queue
+from libcommon.queue.metrics import JobTotalMetricDocument, WorkerSizeJobsCountDocument
 
 from cache_maintenance.queue_metrics import collect_queue_metrics, collect_worker_size_jobs_count
 

jobs/mongodb_migration/src/mongodb_migration/deletion_migrations.py

@@ -5,7 +5,7 @@
 from collections.abc import Mapping
 from typing import Any, Optional
 
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from libcommon.simple_cache import CachedResponseDocument
 from mongoengine.connection import get_db
 

jobs/mongodb_migration/src/mongodb_migration/migrations/_20230126164900_queue_job_add_priority.py

@@ -4,7 +4,7 @@
 import logging
 
 from libcommon.constants import QUEUE_COLLECTION_JOBS, QUEUE_MONGOENGINE_ALIAS
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from mongoengine.connection import get_db
 
 from mongodb_migration.check import check_documents

jobs/mongodb_migration/src/mongodb_migration/migrations/_20230516101500_queue_job_add_revision.py

@@ -4,7 +4,7 @@
 import logging
 
 from libcommon.constants import QUEUE_COLLECTION_JOBS, QUEUE_MONGOENGINE_ALIAS
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from mongoengine.connection import get_db
 
 from mongodb_migration.check import check_documents

jobs/mongodb_migration/src/mongodb_migration/migrations/_20230622131500_lock_add_owner.py

@@ -4,7 +4,7 @@
 import logging
 
 from libcommon.constants import QUEUE_COLLECTION_LOCKS, QUEUE_MONGOENGINE_ALIAS
-from libcommon.queue import Lock
+from libcommon.queue.lock import Lock
 from mongoengine.connection import get_db
 
 from mongodb_migration.check import check_documents

jobs/mongodb_migration/src/mongodb_migration/migrations/_20230705160600_queue_job_add_difficulty.py

@@ -9,7 +9,7 @@
     QUEUE_MONGOENGINE_ALIAS,
 )
 from libcommon.processing_graph import specification
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from mongoengine.connection import get_db
 
 from mongodb_migration.check import check_documents

jobs/mongodb_migration/src/mongodb_migration/migrations/_20230825170200_lock_add_ttl.py

@@ -4,7 +4,7 @@
 import logging
 
 from libcommon.constants import QUEUE_COLLECTION_LOCKS, QUEUE_MONGOENGINE_ALIAS
-from libcommon.queue import Lock
+from libcommon.queue.lock import Lock
 from mongoengine.connection import get_db
 
 from mongodb_migration.check import check_documents

jobs/mongodb_migration/src/mongodb_migration/renaming_migrations.py

@@ -4,7 +4,7 @@
 import logging
 from typing import Optional
 
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from libcommon.simple_cache import CachedResponseDocument
 from mongoengine.connection import get_db
 

jobs/mongodb_migration/tests/migrations/test_20230511100700_queue_delete_indexes_with_force.py

@@ -2,7 +2,7 @@
 # Copyright 2023 The HuggingFace Authors.
 
 from libcommon.constants import QUEUE_COLLECTION_JOBS, QUEUE_MONGOENGINE_ALIAS
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from libcommon.resources import MongoResource
 from libcommon.utils import get_datetime
 from mongoengine.connection import get_db

jobs/mongodb_migration/tests/test_deletion_migrations.py

@@ -9,7 +9,7 @@
     QUEUE_MONGOENGINE_ALIAS,
 )
 from libcommon.dtos import Status
-from libcommon.queue import JobDocument
+from libcommon.queue.jobs import JobDocument
 from libcommon.resources import MongoResource
 from libcommon.utils import get_datetime
 from mongoengine.connection import get_db

libs/libapi/tests/conftest.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 from libcommon.config import CacheConfig, QueueConfig
-from libcommon.queue import _clean_queue_database
+from libcommon.queue.utils import _clean_queue_database
 from libcommon.resources import CacheMongoResource, QueueMongoResource
 from libcommon.simple_cache import _clean_cache_database
 from pytest import MonkeyPatch, TempPathFactory, fixture

libs/libcommon/src/libcommon/orchestrator.py

@@ -20,7 +20,7 @@
 from libcommon.dtos import JobInfo, JobResult, Priority
 from libcommon.processing_graph import ProcessingGraph, ProcessingStep, ProcessingStepDoesNotExist, processing_graph
 from libcommon.prometheus import StepProfiler
-from libcommon.queue import Queue
+from libcommon.queue.jobs import Queue
 from libcommon.simple_cache import (
     CachedArtifactNotFoundError,
     delete_dataset_responses,

libs/libcommon/src/libcommon/prometheus.py

@@ -17,7 +17,7 @@
 from psutil import disk_usage
 
 from libcommon.constants import LONG_DURATION_PROMETHEUS_HISTOGRAM_BUCKETS
-from libcommon.queue import JobTotalMetricDocument, WorkerSizeJobsCountDocument
+from libcommon.queue.metrics import JobTotalMetricDocument, WorkerSizeJobsCountDocument
 from libcommon.simple_cache import CacheTotalMetricDocument
 from libcommon.storage import StrPath
 

libs/libcommon/src/libcommon/queue/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 The HuggingFace Authors.
+