fix: fix tests + some i18n utils (#799)

.github/workflows/pull_request.yml

@@ -21,7 +21,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version:
+        - "3.7"
+        - "3.8"
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -93,7 +95,9 @@ jobs:
       # Launch tests
       #----------------------------------------------
       - name: Configure docker
-        run: echo "OFF_UID=$UID" >> .env
+        run: |
+          echo "OFF_UID=$UID" >> .env
+          echo "PYTHON_VERSION=${{ matrix.python-version }}" >> .env
       - name: Launch tests
         run: make tests
 

Dockerfile

@@ -1,6 +1,8 @@
+ARG PYTHON_VERSION=3.7
+
 # base python setup
 # -----------------
-FROM python:3.7-slim as python-base
+FROM python:$PYTHON_VERSION-slim as python-base
 RUN apt-get update && \
     apt-get install --no-install-suggests --no-install-recommends -y gettext curl build-essential && \
     apt-get autoremove --purge && \

Makefile

@@ -123,14 +123,19 @@ checks: toml-check flake8 black-check mypy isort-check docs
 
 lint: toml-lint isort black
 
-tests: create_external_networks unit-tests integration-tests
+tests: create_external_networks i18n-compile unit-tests integration-tests
 
 quality: lint checks tests
 
 health:
 	@echo "🥫 Running health tests …"
 	@curl --fail --fail-early 127.0.0.1:5500/api/v1/health
 
+i18n-compile:
+	@echo "🥫 Compiling translations …"
+# Note it's important to have --no-deps, to avoid launching a concurrent postgres instance
+	${DOCKER_COMPOSE} run --rm --entrypoint bash --no-deps workers -c "cd i18n && . compile.sh"
+
 unit-tests:
 	@echo "🥫 Running tests …"
 	# run tests in worker to have more memory

docker/dev.yml

@@ -8,6 +8,7 @@ x-robotoff-dev: &robotoff-dev
       args:
         OFF_UID: ${OFF_UID:-1000}
         OFF_GID: ${OFF_GID:-1000}
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.7}
     volumes:
       # make updates to code available
       - ./robotoff:/opt/robotoff/robotoff
@@ -18,6 +19,8 @@ x-robotoff-dev: &robotoff-dev
       # make tests available
       - ./tests:/opt/robotoff/tests
       - ./.cov:/opt/robotoff/.cov
+      # make i18N available
+      - ./i18n:/opt/robotoff/i18n
       # make data available
       - ./data:/opt/robotoff/data
       # make doc generation available

i18n/compile.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-
 for i in */LC_MESSAGES/*.po; do
   msgfmt -o "${i%\.*}.mo" $i
 done

tests/conftest.py

@@ -7,8 +7,11 @@
 
 @pytest.fixture(scope="session")
 def peewee_db_create():
-    with models.db:
-        models.db.create_tables(models.MODELS, safe=True)
+    models.db.close()  # insure creating a new connection
+    with models.db as db:
+        db.create_tables(models.MODELS, safe=True)
+        print("DEBUG: models created ", db.get_tables())
+    yield models.db
 
 
 @pytest.fixture()
@@ -18,3 +21,4 @@ def peewee_db(peewee_db_create):
     # to avoid reusing same transaction next time
     if not models.db.is_closed():
         models.db.rollback()
+        models.db.close()

tests/integration/insights/test_annotate.py

@@ -3,17 +3,15 @@
 from robotoff.insights.annotate import CategoryAnnotator
 from robotoff.models import ProductInsight
 
-from ..models_utils import ProductInsightFactory
+from ..models_utils import ProductInsightFactory, clean_db
 
 
 @pytest.fixture(autouse=True)
 def _set_up_and_tear_down(peewee_db):
-    # clean db
-    ProductInsight.delete().execute()
+    clean_db()
     # Run the test case.
     yield
-    # Tear down.
-    ProductInsight.delete().execute()
+    clean_db()
 
 
 def test_annotation_fails_is_rolledback(mocker):

tests/integration/insights/test_category_import.py

@@ -2,12 +2,11 @@
 
 from robotoff import settings
 from robotoff.insights.importer import import_insights
-from robotoff.models import Prediction as PredictionModel
 from robotoff.models import ProductInsight
 from robotoff.prediction.types import Prediction, PredictionType
 from robotoff.products import Product
 
-from ..models_utils import PredictionFactory, ProductInsightFactory
+from ..models_utils import PredictionFactory, ProductInsightFactory, clean_db
 
 insight_id1 = "94371643-c2bc-4291-a585-af2cb1a5270a"
 barcode1 = "00001"
@@ -16,8 +15,7 @@
 @pytest.fixture(autouse=True)
 def _set_up_and_tear_down(peewee_db):
     # clean db
-    ProductInsight.delete().execute()
-    PredictionModel.delete().execute()
+    clean_db()
     # a category already exists
     PredictionFactory(
         barcode=barcode1,
@@ -33,8 +31,7 @@ def _set_up_and_tear_down(peewee_db):
     # Run the test case.
     yield
     # Tear down.
-    ProductInsight.delete().execute()
-    PredictionModel.delete().execute()
+    clean_db()
 
 
 def matcher_prediction(category):

tests/integration/insights/test_process_insights.py

@@ -6,17 +6,17 @@
 from robotoff.models import ProductInsight
 from robotoff.scheduler import process_insights
 
-from ..models_utils import ProductInsightFactory
+from ..models_utils import ProductInsightFactory, clean_db
 
 
 @pytest.fixture(autouse=True)
 def _set_up_and_tear_down(peewee_db):
     # clean db
-    ProductInsight.delete().execute()
+    clean_db()
     # Run the test case.
     yield
     # Tear down.
-    ProductInsight.delete().execute()
+    clean_db()
 
 
 # global for generating items

tests/integration/models_utils.py

@@ -10,7 +10,7 @@
 import factory
 from factory_peewee import PeeweeModelFactory
 
-from robotoff import settings
+from robotoff import models, settings
 from robotoff.models import (
     AnnotationVote,
     ImageModel,
@@ -128,14 +128,16 @@ class Meta:
 
 
 def clean_db():
+    print("DEBUG: Before cleaning: ", models.db.get_tables())
     # remove all models
     for model in (
         AnnotationVote,
-        ImageModel,
-        ImagePrediction,
         LogoAnnotation,
+        ImagePrediction,
+        ImageModel,
         LogoConfidenceThreshold,
         Prediction,
         ProductInsight,
     ):
-        model.delete()
+        model.delete().execute()
+    print("DEBUG: After cleaning: ", models.db.get_tables())

tests/integration/test_annotate_image.py

@@ -76,7 +76,9 @@ def fetch_taxonomy_mock(url: str, *args, **kwargs):
 
 
 def test_image_brand_annotation(client, monkeypatch, fake_taxonomy):
-    ann = LogoAnnotationFactory(image_prediction__image__source_image="/images/2.jpg")
+    ann = LogoAnnotationFactory(
+        image_prediction__image__source_image="/images/2.jpg", annotation_type="brand"
+    )
     barcode = ann.image_prediction.image.barcode
     _fake_store(monkeypatch, barcode)
     monkeypatch.setattr(

tests/integration/test_api.py

@@ -8,25 +8,20 @@
 from robotoff.app.api import api
 from robotoff.models import AnnotationVote, ProductInsight
 
-from .models_utils import AnnotationVoteFactory, ProductInsightFactory
+from .models_utils import AnnotationVoteFactory, ProductInsightFactory, clean_db
 
 insight_id = "94371643-c2bc-4291-a585-af2cb1a5270a"
 
 
 @pytest.fixture(autouse=True)
 def _set_up_and_tear_down(peewee_db):
     # clean db
-    AnnotationVote.delete().execute()
-    ProductInsight.delete().execute()
+    clean_db()
     # Set up.
     ProductInsightFactory(id=insight_id, barcode=1)
-
     # Run the test case.
     yield
-
-    # Tear down.
-    AnnotationVote.delete().execute()
-    ProductInsight.delete().execute()
+    clean_db()
 
 
 @pytest.fixture()
@@ -95,7 +90,7 @@ def test_popular_question(client, mocker):
 
 def test_popular_question_pagination(client, mocker):
     mocker.patch("robotoff.insights.question.get_product", return_value={})
-    ProductInsight.delete().execute()
+    ProductInsight.delete().execute()  # remove default sample
     for i in range(0, 12):
         ProductInsightFactory(barcode=i, unique_scans_n=100 - i)
 

tests/integration/test_models_integration.py

@@ -2,19 +2,17 @@
 
 from robotoff.models import AnnotationVote, ProductInsight
 
-from .models_utils import AnnotationVoteFactory, ProductInsightFactory
+from .models_utils import AnnotationVoteFactory, ProductInsightFactory, clean_db
 
 
 @pytest.fixture(autouse=True)
 def _set_up_and_tear_down(peewee_db):
     # clean db
-    AnnotationVote.delete().execute()
-    ProductInsight.delete().execute()
+    clean_db()
     # Run the test case.
     yield
     # Tear down.
-    AnnotationVote.delete().execute()
-    ProductInsight.delete().execute()
+    clean_db()
 
 
 def test_vote_cascade_on_insight_deletion(peewee_db):