Skip to content

Commit

Permalink
fix: improve add-logo-to-ann CLI command
Browse files Browse the repository at this point in the history
  • Loading branch information
raphael0202 committed Dec 22, 2022
1 parent 9cfb410 commit a6c6fc7
Showing 1 changed file with 9 additions and 4 deletions.
13 changes: 9 additions & 4 deletions robotoff/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ def add_logo_to_ann(
)
) -> None:
"""Index all missing logos in Elasticsearch ANN index."""
import logging
import time
from itertools import groupby

Expand All @@ -432,9 +433,12 @@ def add_logo_to_ann(
from robotoff.utils import get_logger

logger = get_logger()
seen = get_stored_logo_ids()
logging.getLogger("elastic_transport.transport").setLevel(logging.WARNING)

seen = get_stored_logo_ids()
added = 0
with db:
logger.info("Fetching logo embedding to index...")
logo_embedding_iter = tqdm.tqdm(
LogoEmbedding.select(LogoEmbedding, LogoAnnotation, ImageModel.id)
.join(LogoAnnotation)
Expand All @@ -443,7 +447,7 @@ def add_logo_to_ann(
.order_by(ImageModel.id)
.iterator()
)
for image_id, logo_embedding_batch in groupby(
for _, logo_embedding_batch in groupby(
logo_embedding_iter, lambda x: x.logo.image_prediction.image.id
):
logo_embeddings = list(logo_embedding_batch)
Expand All @@ -456,7 +460,6 @@ def add_logo_to_ann(
if not to_process:
continue

logger.info(f"Adding logos of image {image_id} ({len(to_process)} to add)")
try:
add_logos_to_ann(to_process)
except (
Expand All @@ -465,11 +468,13 @@ def add_logo_to_ann(
) as e:
logger.info("Request error during logo addition to ANN", exc_info=e)

logger.info(f"Logos of image {image_id} were added.")
added += len(to_process)

if sleep_time:
time.sleep(sleep_time)

logger.info(f"{added} embeddings indexed")


@app.command()
def import_embedding(
Expand Down

0 comments on commit a6c6fc7

Please sign in to comment.