containerize application

wmo-im · Mar 18, 2024 · c344ca0 · c344ca0
1 parent 1135350
commit c344ca0
Show file tree

Hide file tree

Showing 45 changed files with 240 additions and 235 deletions.
diff --git a/Makefile b/Makefile
@@ -19,7 +19,7 @@
 #
 ###############################################################################
 
-DOCKER_COMPOSE_ARGS=--project-name wis2-gdc --file docker/docker-compose.yml --file docker/docker-compose.override.yml
+DOCKER_COMPOSE_ARGS=--project-name wis2-gdc --file docker-compose.yml --file docker-compose.override.yml
 
 build:
 	docker compose $(DOCKER_COMPOSE_ARGS) build
@@ -28,7 +28,7 @@ up:
 	docker compose $(DOCKER_COMPOSE_ARGS) up --detach
 
 dev:
-	docker compose $(DOCKER_COMPOSE_ARGS) --file docker/docker-compose.dev.yml up --detach
+	docker compose $(DOCKER_COMPOSE_ARGS) --file docker-compose.dev.yml up --detach
 
 login:
 	docker exec -it wis2-gdc-management /bin/bash

diff --git a/README.md b/README.md
@@ -84,7 +84,19 @@ wis2-gdc archive foo.zip
 
 ### Docker
 
-Instructions to run wis2-gdc via Docker can be found the [`docker`](docker) directory.
+The Docker setup uses Docker and Docker Compose to manage the following services:
+
+- **wis2-gdc-api**: GDC API powered by [pygeoapi](https://pygeoapi.io)
+- **Elasticsearch**: GDC search engine backend
+- **wis2-gdc-management**: management service to ingest, validate and publish discovery metadata published from a WIS2 Global Broker instance
+  - the default Global Broker connection is to Météo-France.  This can be modified in `pywis-pubsub.yml` to point to an alternate Global Broker
+
+See [`wis2-gdc.env`](wis2-gdc.env) for default environment variable settings.
+
+To adjust service ports, edit [`docker-compose.override.yml`](docker-compose.override.yml) accordingly.
+
+The [`Makefile`](Makefile) in the root directory provides options to manage the Docker Compose setup.
+
 
 ## Development
 

diff --git a/docker/docker-compose.dev.yml → docker-compose.dev.yml b/docker/docker-compose.dev.yml → docker-compose.dev.yml
diff --git a/docker/docker-compose.override.yml → docker-compose.override.yml b/docker/docker-compose.override.yml → docker-compose.override.yml
@@ -20,10 +20,13 @@
 ###############################################################################
 
 services:
-  mosquitto:
+  wis2-gdc-broker:
     ports:
       - 1883:1883  # default
       - 1884:1884  # websockets
+  wis2-gdc-metrics-collector:
+    ports:
+      - 8001:8001
   wis2-gdc-api:
     ports:
       - 80:80
diff --git a/docker/docker-compose.yml → docker-compose.yml b/docker/docker-compose.yml → docker-compose.yml
@@ -50,20 +50,32 @@ services:
     networks:
       - wis2-gdc-net
 
-  mosquitto:
-    container_name: mosquitto
+  wis2-gdc-broker:
+    container_name: wis2-gdc-broker
     restart: always
     build:
-      context: ./mosquitto
+      context: ./wis2-gdc-broker/
     env_file:
       - wis2-gdc.env
     networks:
       - wis2-gdc-net
 
+  wis2-gdc-metrics-collector:
+    container_name: wis2-gdc-metrics-collector
+    restart: unless-stopped
+    build:
+      context: ./wis2-gdc-metrics-collector/
+    env_file:
+      - wis2-gdc.env
+    networks:
+      - wis2-gdc-net
+    depends_on:
+      - wis2-gdc-broker
+
   wis2-gdc-management:
     container_name: wis2-gdc-management
     build:
-      context: ..
+      context: ./wis2-gdc-management/
     env_file:
       - wis2-gdc.env
     environment:
@@ -85,7 +97,7 @@ services:
   wis2-gdc-api:
     container_name: wis2-gdc-api
     build:
-      context: wis2-gdc-api/
+      context: ./wis2-gdc-api/
     image: geopython/pygeoapi:latest
     depends_on:
       wis2-gdc-management:

diff --git a/docker/README.md b/docker/README.md
diff --git a/docker/wis2-gdc.env b/docker/wis2-gdc.env
diff --git a/pywis-pubsub.yml b/pywis-pubsub.yml
diff --git a/docker/wis2-gdc-api/Dockerfile → wis2-gdc-api/Dockerfile b/docker/wis2-gdc-api/Dockerfile → wis2-gdc-api/Dockerfile
diff --git a/docker/wis2-gdc-api/app.py → wis2-gdc-api/app.py b/docker/wis2-gdc-api/app.py → wis2-gdc-api/app.py
diff --git a/docker/wis2-gdc-api/entrypoint.sh → wis2-gdc-api/entrypoint.sh b/docker/wis2-gdc-api/entrypoint.sh → wis2-gdc-api/entrypoint.sh
diff --git a/docker/wis2-gdc-api/wis2-gdc-api.yml → wis2-gdc-api/wis2-gdc-api.yml b/docker/wis2-gdc-api/wis2-gdc-api.yml → wis2-gdc-api/wis2-gdc-api.yml
diff --git a/docker/mosquitto/Dockerfile → wis2-gdc-broker/Dockerfile b/docker/mosquitto/Dockerfile → wis2-gdc-broker/Dockerfile
diff --git a/docker/mosquitto/acl.conf → wis2-gdc-broker/acl.conf b/docker/mosquitto/acl.conf → wis2-gdc-broker/acl.conf
diff --git a/docker/mosquitto/entrypoint.sh → wis2-gdc-broker/entrypoint.sh b/docker/mosquitto/entrypoint.sh → wis2-gdc-broker/entrypoint.sh
diff --git a/docker/mosquitto/mosquitto.conf → wis2-gdc-broker/mosquitto.conf b/docker/mosquitto/mosquitto.conf → wis2-gdc-broker/mosquitto.conf
diff --git a/Dockerfile → wis2-gdc-management/Dockerfile b/Dockerfile → wis2-gdc-management/Dockerfile
diff --git a/MANIFEST.in → wis2-gdc-management/MANIFEST.in b/MANIFEST.in → wis2-gdc-management/MANIFEST.in
diff --git a/wis2-gdc-management/README.md b/wis2-gdc-management/README.md
@@ -0,0 +1,3 @@
+# wis2-gdc-management
+
+Python package to perform WIS2 GDC management functions.
diff --git a/docker/entrypoint.sh → wis2-gdc-management/docker/entrypoint.sh b/docker/entrypoint.sh → wis2-gdc-management/docker/entrypoint.sh
diff --git a/docker/pywis-pubsub.yml → wis2-gdc-management/docker/pywis-pubsub.yml b/docker/pywis-pubsub.yml → wis2-gdc-management/docker/pywis-pubsub.yml
diff --git a/docker/wis2-gdc-management.cron → ...anagement/docker/wis2-gdc-management.cron b/docker/wis2-gdc-management.cron → ...anagement/docker/wis2-gdc-management.cron
diff --git a/requirements-backend.txt → wis2-gdc-management/requirements-backend.txt b/requirements-backend.txt → wis2-gdc-management/requirements-backend.txt
diff --git a/requirements-dev.txt → wis2-gdc-management/requirements-dev.txt b/requirements-dev.txt → wis2-gdc-management/requirements-dev.txt
diff --git a/requirements.txt → wis2-gdc-management/requirements.txt b/requirements.txt → wis2-gdc-management/requirements.txt
diff --git a/setup.py → wis2-gdc-management/setup.py b/setup.py → wis2-gdc-management/setup.py
diff --git a/wis2_gdc/__init__.py → wis2-gdc-management/wis2_gdc/__init__.py b/wis2_gdc/__init__.py → wis2-gdc-management/wis2_gdc/__init__.py
diff --git a/wis2_gdc/archive.py → wis2-gdc-management/wis2_gdc/archive.py b/wis2_gdc/archive.py → wis2-gdc-management/wis2_gdc/archive.py
diff --git a/wis2_gdc/backend/__init__.py → ...c-management/wis2_gdc/backend/__init__.py b/wis2_gdc/backend/__init__.py → ...c-management/wis2_gdc/backend/__init__.py
diff --git a/wis2_gdc/backend/base.py → wis2-gdc-management/wis2_gdc/backend/base.py b/wis2_gdc/backend/base.py → wis2-gdc-management/wis2_gdc/backend/base.py
diff --git a/wis2_gdc/backend/elastic.py → ...dc-management/wis2_gdc/backend/elastic.py b/wis2_gdc/backend/elastic.py → ...dc-management/wis2_gdc/backend/elastic.py
diff --git a/wis2_gdc/backend/ogcapi_records.py → ...gement/wis2_gdc/backend/ogcapi_records.py b/wis2_gdc/backend/ogcapi_records.py → ...gement/wis2_gdc/backend/ogcapi_records.py
diff --git a/wis2_gdc/env.py → wis2-gdc-management/wis2_gdc/env.py b/wis2_gdc/env.py → wis2-gdc-management/wis2_gdc/env.py
@@ -51,16 +51,14 @@ def str2bool(value: Any) -> bool:
 CENTRE_ID = os.environ.get('WIS2_GDC_CENTRE_ID')
 GB = os.environ.get('WIS2_GDC_GB')
 GB_TOPIC = os.environ.get('WIS2_GDC_GB_TOPIC')
-OPENMETRICS_FILE = os.environ.get('WIS2_GDC_OPENMETRICS_FILE')
 PUBLISH_REPORTS = str2bool(os.environ.get('WIS2_GDC_PUBLISH_REPORTS', 'false'))
 REJECT_ON_FAILING_ETS = str2bool(os.environ.get('WIS2_GDC_REJECT_ON_FAILING_ETS', 'true'))  # noqa
 RUN_KPI = str2bool(os.environ.get('WIS2_GDC_RUN_KPI', 'false'))
 
 GB_LINKS = []
 
-if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE,
-            BACKEND_CONNECTION, BROKER_URL, CENTRE_ID,
-            GB, GB_TOPIC, OPENMETRICS_FILE]:
+if None in [API_URL, API_URL_DOCKER, BACKEND_TYPE, BACKEND_CONNECTION,
+            BROKER_URL, CENTRE_ID, GB, GB_TOPIC]:
     raise EnvironmentError('Environment variables not set!')
 
 for key, value in os.environ.items():

diff --git a/wis2_gdc/harvester/__init__.py → ...management/wis2_gdc/harvester/__init__.py b/wis2_gdc/harvester/__init__.py → ...management/wis2_gdc/harvester/__init__.py
diff --git a/wis2_gdc/harvester/base.py → ...gdc-management/wis2_gdc/harvester/base.py b/wis2_gdc/harvester/base.py → ...gdc-management/wis2_gdc/harvester/base.py
diff --git a/wis2_gdc/harvester/wis2box.py → ...-management/wis2_gdc/harvester/wis2box.py b/wis2_gdc/harvester/wis2box.py → ...-management/wis2_gdc/harvester/wis2box.py
diff --git a/wis2_gdc/hook.py → wis2-gdc-management/wis2_gdc/hook.py b/wis2_gdc/hook.py → wis2-gdc-management/wis2_gdc/hook.py
diff --git a/wis2_gdc/monitor/__init__.py → ...c-management/wis2_gdc/monitor/__init__.py b/wis2_gdc/monitor/__init__.py → ...c-management/wis2_gdc/monitor/__init__.py
diff --git a/wis2_gdc/registrar.py → wis2-gdc-management/wis2_gdc/registrar.py b/wis2_gdc/registrar.py → wis2-gdc-management/wis2_gdc/registrar.py
@@ -36,7 +36,6 @@
 from wis2_gdc.env import (BACKEND_TYPE, BACKEND_CONNECTION, BROKER_URL,
                           CENTRE_ID, GB_LINKS, PUBLISH_REPORTS,
                           REJECT_ON_FAILING_ETS, RUN_KPI)
-from wis2_gdc.monitor.metrics import Metrics
 
 LOGGER = logging.getLogger(__name__)
 
@@ -51,7 +50,6 @@ def __init__(self):
 
         self.broker = None
         self.metadata = None
-        self.metrics = Metrics()
 
         if PUBLISH_REPORTS:
             self.broker = MQTTPubSubClient(BROKER_URL)
@@ -108,19 +106,18 @@ def register(self, metadata: dict) -> None:
                     LOGGER.warning('ETS errors; metadata not published')
                     return
             except KeyError:  # validation error
-                self.metrics.failed_total.labels(*centre_id_labels).inc()
                 LOGGER.debug('Validation errors; metadata not published')
-                self.metrics.write()
+                self.broker.pub('wis2-gdc/metrics/failed_total',
+                                json.dumps(centre_id_labels))
                 return
 
-        self.metrics.passed_total.labels(*centre_id_labels).inc()
+        self.broker.pub('wis2-gdc/metrics/passed_total',
+                        json.dumps(centre_id_labels))
 
-        # TODO: remove following wis2box b7 updates
         data_policy = self.metadata['properties']['wmo:dataPolicy']
-        if data_policy == 'core':
-            self.metrics.core_total.labels(*centre_id_labels).inc()
-        elif data_policy == 'recommended':
-            self.metrics.recommended_total.labels(*centre_id_labels).inc()
+
+        self.broker.pub(f'wis2-gdc/metrics/{data_policy}_total',
+                        json.dumps(centre_id_labels))
 
         LOGGER.info('Updating links')
         self.update_record_links()
@@ -138,8 +135,6 @@ def register(self, metadata: dict) -> None:
                 LOGGER.info('Publishing KPI report to broker')
                 self.broker.pub(topic, json.dumps(kpi_results))
 
-        self.metrics.write()
-
     def _run_ets(self) -> dict:
         """
         Helper function to run ETS

diff --git a/wis2_gdc/sync.py → wis2-gdc-management/wis2_gdc/sync.py b/wis2_gdc/sync.py → wis2-gdc-management/wis2_gdc/sync.py
diff --git a/wis2-gdc-metrics-collector/Dockerfile b/wis2-gdc-metrics-collector/Dockerfile
@@ -0,0 +1,31 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+FROM python:3-alpine
+
+COPY . /app
+RUN pip install -r /app/requirements.txt
+
+EXPOSE 8001
+
+ENV PYTHONUNBUFFERED="true"
+
+ENTRYPOINT [ "python3","-u","/app/metrics_collector.py" ]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# wis2-gdc-management

		Python package to perform WIS2 GDC management functions.