diff --git a/.env b/.env index f2496a51dc..9ac302bba9 100644 --- a/.env +++ b/.env @@ -1,10 +1,10 @@ COMPOSE_PATH_SEPARATOR=; -COMPOSE_FILE=docker-compose.yml;docker/dev.yml;docker/mongodb.yml;docker/ml.yml +COMPOSE_FILE=docker-compose.yml;docker/dev.yml;docker/ml.yml # Docker RESTART_POLICY=no # for dev only: connection to local product opener network -PO_LOCAL_NET=po_default +COMMON_NET_NAME=po_default # User uid - in dev align this with your own user uid / gid # see https://gist.github.com/alexgarel/6e6158ee869d6db2192e0441fd58576e @@ -39,7 +39,6 @@ ELASTIC_USER=elastic ELASTIC_PASSWORD=elastic # Postgres -# use network qualified name for dev for we have two networks POSTGRES_HOST=postgres POSTGRES_DB=postgres POSTGRES_USER=postgres @@ -61,13 +60,16 @@ INFLUXDB_AUTH_TOKEN= # on linux, this will work if you have an influxdb listening on 0.0.0.0 # INFLUXDB_HOST=host.docker.internal -# MongoDB (dev settings, using robotoff-specific MongoDB instance by default) +# MongoDB (dev settings, using shared MongoDB instance by default) # To use Product Opener instance, use following commented line # MONGO_URI=mongodb://mongodb.po_default:27017 MONGO_URI=mongodb://mongodb:27017 -# Redis +# Local Redis, used to manage rq jobs REDIS_HOST=redis +# Redis where Product Opener publishes updates in the Redis Stream +# We uses here the same Redis instance as the one used for rq jobs +# for simplicity, but in production it uses a different one. REDIS_UPDATE_HOST=redis # OpenFoodFacts API diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 98160a90eb..98392d3296 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -31,9 +31,6 @@ jobs: echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV # In staging, we use the Redis of shared services to get Product Update echo "REDIS_UPDATE_HOST=redis.po_webnet" >> $GITHUB_ENV - echo "REDIS_UPDATE_PORT=6379" >> $GITHUB_ENV - # And the local redis for the rest - echo "REDIS_HOST=redis.robotoff_default" >> $GITHUB_ENV echo "INFLUXDB_HOST=10.1.0.200" >> $GITHUB_ENV echo "IMAGE_MODERATION_SERVICE_URL=https://nutripatrol.openfoodfacts.net/api/v1/flags" >> $GITHUB_ENV - name: Set various variable for production deployment @@ -46,7 +43,6 @@ jobs: echo "MONGO_URI=mongodb://10.1.0.113:27017" >> $GITHUB_ENV # In prod, we use the Redis running in VM echo "REDIS_UPDATE_HOST=10.1.0.113" >> $GITHUB_ENV - echo "REDIS_UPDATE_PORT=6379" >> $GITHUB_ENV echo "INFLUXDB_HOST=10.1.0.201" >> $GITHUB_ENV echo "IMAGE_MODERATION_SERVICE_URL=https://nutripatrol.openfoodfacts.org/api/v1/flags" >> $GITHUB_ENV - name: Wait for container build workflow @@ -123,7 +119,15 @@ jobs: # Set app variables echo "ROBOTOFF_INSTANCE=${{ env.ROBOTOFF_INSTANCE }}" >> .env echo "ROBOTOFF_TLD=${{ env.ROBOTOFF_TLD }}" >> .env + # This is the network shared with Product Opener. + # In staging, Product Opener is deployed on the same VM. + # In production Product Opener is deployed on a separate server and is not dockerized. + echo "COMMON_NET_NAME=po_webnet" >> .env echo "REDIS_HOST=redis.robotoff_default" >> .env + echo "REDIS_UPDATE_HOST=${{ env.REDIS_UPDATE_HOST }}" >> .env + echo "REDIS_UPDATE_PORT=6379" >> .env + # We use local PostgreSQL instance with `robotoff_default` network, + # so that there is no possibility of conflict with PO PostgreSQL instance. echo "POSTGRES_HOST=postgres.robotoff_default" >> .env echo "POSTGRES_DB=postgres" >> .env echo "POSTGRES_USER=postgres" >> .env diff --git a/Makefile b/Makefile index 072be6e538..a070ae9f28 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ ENV_FILE ?= .env MOUNT_POINT ?= /mnt HOSTS=127.0.0.1 robotoff.openfoodfacts.localhost DOCKER_COMPOSE=docker compose --env-file=${ENV_FILE} -DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=robotoff_test PO_LOCAL_NET=po_test docker compose --env-file=${ENV_FILE} +DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=robotoff_test COMMON_NET_NAME=po_test docker compose --env-file=${ENV_FILE} ML_OBJECT_DETECTION_MODELS := tf-universal-logo-detector tf-nutrition-table tf-nutriscore # mount information for robotoff backup ZFS dataset @@ -194,7 +194,7 @@ docs: @echo "🥫 Generationg doc…" ${DOCKER_COMPOSE} run --rm --no-deps api ./build_mkdocs.sh -checks: toml-check flake8 black-check mypy isort-check docs +checks: create_external_networks toml-check flake8 black-check mypy isort-check docs lint: toml-lint isort black @@ -250,7 +250,7 @@ create_external_volumes: create_external_networks: @echo "🥫 Creating external networks if needed … (dev only)" - ( docker network create ${PO_LOCAL_NET} || true ) + ( docker network create ${COMMON_NET_NAME} || true ) # for tests ( docker network create po_test || true ) diff --git a/doc/how-to-guides/deployment/dev-install.md b/doc/how-to-guides/deployment/dev-install.md index d0c7de491c..143d7edc0c 100644 --- a/doc/how-to-guides/deployment/dev-install.md +++ b/doc/how-to-guides/deployment/dev-install.md @@ -11,8 +11,6 @@ You should, consider those changes: You should also run `make dl-models` to download all machine learning models locally. -- if you don't work on Product Opener, you should launch MongoDB by adding `docker/mongodb.yml` to `COMPOSE_FILE` - - change `OFF_UID` and `OFF_GID` to match your own user UID/GID (optional, only if you experience some file permission issue, see [Getting developper uid for docker](https://gist.github.com/alexgarel/6e6158ee869d6db2192e0441fd58576e)) Note: **beware** not to commit your local changes to `.env` file! diff --git a/docker-compose.yml b/docker-compose.yml index 42d78eaeab..e1a3fe981f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,9 @@ x-robotoff-base: # this image is built with target=runtime, so it won't have the dev dependencies image: ghcr.io/openfoodfacts/robotoff:${TAG} volumes: *robotoff-base-volumes + networks: + - default + - common_net x-robotoff-base-env: &robotoff-base-env @@ -61,6 +64,10 @@ x-robotoff-worker-base: - postgres - redis mem_limit: 8g + networks: + - default + - common_net + services: api: <<: *robotoff-base @@ -168,3 +175,8 @@ volumes: networks: default: + + # this is the network shared with product opener + common_net: + name: ${COMMON_NET_NAME} + external: true diff --git a/docker/mongodb.yml b/docker/mongodb.yml deleted file mode 100644 index bf2073a338..0000000000 --- a/docker/mongodb.yml +++ /dev/null @@ -1,11 +0,0 @@ -services: - mongodb: - image: mongo:4.4 - command: mongod --wiredTigerCacheSizeGB 1.5 - ports: - - "127.0.0.1:${MONGO_EXPOSE_PORT:-27017}:27017" - volumes: - - mongodb_data:/data/db - -volumes: - mongodb_data: diff --git a/docker/prod.yml b/docker/prod.yml index 7977ee7341..9b32521812 100644 --- a/docker/prod.yml +++ b/docker/prod.yml @@ -8,3 +8,9 @@ volumes: es-data: external: true name: ${COMPOSE_PROJECT_NAME:-robotoff}_es-data + + +networks: + common_net: + name: ${COMMON_NET_NAME} + external: true