From 36b675e4c0d3bf6586cb9b990c7db924ef03a7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bournhonesque?= Date: Thu, 15 Dec 2022 17:48:57 +0100 Subject: [PATCH] fix: improve request exception handling --- robotoff/insights/annotate.py | 11 ++++++----- robotoff/logos.py | 8 ++++++-- robotoff/metrics.py | 12 +++++++----- robotoff/utils/__init__.py | 17 +++++++++++------ robotoff/workers/tasks/import_image.py | 12 ++---------- 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/robotoff/insights/annotate.py b/robotoff/insights/annotate.py index 9313f7ab8d..26c549a6a7 100644 --- a/robotoff/insights/annotate.py +++ b/robotoff/insights/annotate.py @@ -118,15 +118,16 @@ def annotate( ) except HTTPError as e: if e.response.status_code >= 500: - logger.info("HTTPError occurred during OFF update: %s", e) + logger.info( + "HTTPError occurred during OFF update: %s", + e.response.status_code, + ) logger.info("Rolling back SQL transaction") tx.rollback() return FAILED_UPDATE_RESULT raise e - except (Timeout, SSLError) as e: - logger.info( - "Error occurred during OFF update: %s, %s", type(e).__name__, e - ) + except (ConnectionError, Timeout, SSLError) as e: + logger.info("Error occurred during OFF update", exc_info=e) logger.info("Rolling back SQL transaction") tx.rollback() return FAILED_UPDATE_RESULT diff --git a/robotoff/logos.py b/robotoff/logos.py index 2fc80f72a6..00f41fc806 100644 --- a/robotoff/logos.py +++ b/robotoff/logos.py @@ -1,3 +1,4 @@ +import logging import operator from typing import Optional @@ -101,8 +102,11 @@ def get_stored_logo_ids() -> set[int]: ) if not r.ok: - logger.warning( - f"error while fetching stored logo IDs ({r.status_code}): %s", r.text + logger.log( + logging.INFO if r.status_code >= 500 else logging.WARNING, + "error while fetching stored logo IDs (%s): %s", + r.status_code, + r.text, ) return set() diff --git a/robotoff/metrics.py b/robotoff/metrics.py index f5303a375f..81a94dbae9 100644 --- a/robotoff/metrics.py +++ b/robotoff/metrics.py @@ -1,4 +1,5 @@ import datetime +import logging from typing import Optional from urllib.parse import urlparse @@ -155,14 +156,15 @@ def generate_metrics_from_path( try: r = http_session.get(url, timeout=60, auth=settings._off_request_auth) except (ConnectionError, SSLError, Timeout) as e: - logger.info( - "Error during metrics retrieval: url=%s, %s, %s", url, type(e).__name__, e - ) + logger.info("Error during metrics retrieval: url=%s", url, exc_info=e) return inserts if not r.ok: - logger.info( - "HTTPError during metrics retrieval: url=%s, %s", url, r.status_code + logger.log( + logging.INFO if r.status_code >= 500 else logging.WARNING, + "HTTPError during metrics retrieval: url=%s, %s", + url, + r.status_code, ) return inserts diff --git a/robotoff/utils/__init__.py b/robotoff/utils/__init__.py index 29d4732e2a..1a02a9f970 100644 --- a/robotoff/utils/__init__.py +++ b/robotoff/utils/__init__.py @@ -183,17 +183,22 @@ def get_image_from_url( else: r = requests.get(image_url) except (ConnectionError, SSLError, Timeout) as e: - error_message = f"Cannot download image {image_url}: {type(e).__name__}, {e}" + error_message = "Cannot download image %s" if error_raise: - raise ImageLoadingException(error_message) from e - logger.info(error_message) + raise ImageLoadingException(error_message % image_url) from e + logger.info(error_message, image_url, exc_info=e) return None if not r.ok: - error_message = f"Cannot download image {image_url}: HTTP {r.status_code}" + error_message = "Cannot download image %s: HTTP %s" + error_args = (image_url, r.status_code) if error_raise: - raise ImageLoadingException(error_message) - logger.info(error_message) + raise ImageLoadingException(error_message % error_args) + logger.log( + logging.INFO if r.status_code >= 500 else logging.WARNING, + error_message, + *error_args, + ) return None try: diff --git a/robotoff/workers/tasks/import_image.py b/robotoff/workers/tasks/import_image.py index 89ee0f5bbf..95c44caf35 100644 --- a/robotoff/workers/tasks/import_image.py +++ b/robotoff/workers/tasks/import_image.py @@ -371,21 +371,13 @@ def process_created_logos(image_prediction_id: int, server_domain: str): try: add_logos_to_ann(image_instance, logos) except (ConnectionError, HTTPError, Timeout) as e: - logger.info( - "Request error during logo addition to ANN: %s, %s", - type(e).__name__, - e, - ) + logger.info("Request error during logo addition to ANN", exc_info=e) return try: save_nearest_neighbors(logos) except (ConnectionError, HTTPError, Timeout) as e: - logger.info( - "Request error during ANN batch query: %s, %s", - type(e).__name__, - e, - ) + logger.info("Request error during ANN batch query", exc_info=e) return thresholds = get_logo_confidence_thresholds()