Docker for search service, rearchitecture

.env

@@ -0,0 +1,20 @@
+# Password for the 'elastic' user (at least 6 characters)
+# This needs to be set in the environment variables
+# ELASTIC_PASSWORD=
+
+# Version of Elastic products
+STACK_VERSION=8.3.3
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+#ES_PORT=127.0.0.1:9200
+
+# Increase or decrease based on the available host memory (in bytes)
+# 1GB works well, 2GB and above leads to lower latency
+MEM_LIMIT=2147483648

.gitignore

@@ -102,7 +102,6 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
 env/
 venv/

Dockerfile

@@ -0,0 +1,7 @@
+# Instructions from https://fastapi.tiangolo.com/deployment/docker/
+FROM python:3.9
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY ./app /code/app
+CMD ["uvicorn", "app.api:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -9,30 +9,47 @@ The main file is `api.py`, and the Product schema is in `models/product.py`.
 The `scripts/` directory contains various scripts for manual validation, constructing the product schema, importing, etc.
 
 ### Running locally
+Firstly, make sure your environment is configured:
+```commandline
+export ELASTIC_PASSWORD=PASSWORD_HERE
+```
+
+Then start docker:
+```console
+docker-compose up -d
+```
+
 Docker spins up:
 - Two elasticsearch nodes
 - [Elasticvue](https://elasticvue.com/)
+- The search service on port 8000
 
 You will then need to import from CSV (see instructions below).
 
-Make sure your environment is configured:
-```commandline
-export ELASTIC_PASSWORD=PASSWORD_HERE
-```
-
-
-### Helpful commands:
+### Development
+For development, you have two options for running the service:
+1. Docker
+2. Locally
 
-To start docker:
+To develop on docker, make the changes you need, then build the image and compose by running:
 ```console
+docker build -t off_search_image .
 docker-compose up -d
 ```
 
-To start server:
+However, this tends to be slower than developing locally.
+
+To develop locally, create a venv, install dependencies, then run the service:
 ```console
-uvicorn api:app --reload
+virtualenv .
+source venv/bin/activate
+pip install -r requirements.txt
+uvicorn app.api:app --reload --port=8001
 ```
+Note that it's important to use port 8001, as port 8000 will be used by the docker version of the search service. 
 
+
+### Helpful commands:
 To import data from the [CSV export](https://world.openfoodfacts.org/data):
 ```console
 python scripts/perform_import.py --filename=/path/to/file.csv

api.py → app/api.py

@@ -1,9 +1,9 @@
 from elasticsearch_dsl import Q
 from fastapi import FastAPI, HTTPException
 
-from models.product import Product
-from models.request import AutocompleteRequest, SearchRequest
-from utils import connection, constants, response
+from app.models.product import Product
+from app.models.request import AutocompleteRequest, SearchRequest
+from app.utils import connection, constants, response
 
 app = FastAPI()
 connection.get_connection()

models/product.py → app/models/product.py

@@ -1,7 +1,7 @@
 from elasticsearch_dsl import Document, Date, Double, Keyword, Text, Integer
 
-from utils import constants
-from utils.analyzers import autocomplete
+from app.utils import constants
+from app.utils.analyzers import autocomplete
 
 
 class Product(Document):

models/request.py → app/models/request.py

@@ -2,7 +2,7 @@
 from typing import Optional, List, Set
 from pydantic import BaseModel
 
-from utils import constants
+from app.utils import constants
 
 
 class SearchBase(BaseModel):

scripts/es_query.py → app/scripts/es_query.py

@@ -3,8 +3,8 @@
 """
 import time
 
-from models.product import Product
-from utils import connection
+from app.models.product import Product
+from app.utils import connection
 
 
 def manual_query():

scripts/generate_product_from_data_fields.py → app/scripts/generate_product_from_data_fields.py

@@ -2,7 +2,7 @@
 This script takes the data-fields.txt and generates the updated product fields.
 Note, if field names are changed, etc, this will have considerable implications for the index.
 """
-from utils import constants
+from app.utils import constants
 
 
 def get_type_for_field(field):

scripts/http_query.py → app/scripts/http_query.py

@@ -5,8 +5,7 @@
 import time
 import requests
 
-from models.product import Product
-from utils import connection
+from app.utils import connection
 
 
 def manual_query():

scripts/perform_import.py → app/scripts/perform_import.py

@@ -13,8 +13,8 @@
 
 from elasticsearch.helpers import bulk
 
-from models.product import Product
-from utils import connection, constants
+from app.models.product import Product
+from app.utils import connection, constants
 
 
 def gen_documents(filename, next_index):

utils/analyzers.py → app/utils/analyzers.py

@@ -5,4 +5,4 @@
     'autocomplete',
     tokenizer=tokenizer('bigram', 'edge_ngram', min_gram=2, max_gram=25, token_chars=['letter', 'digit', 'punctuation']),
     filter=['lowercase', 'asciifolding']
-)
+)

app/utils/connection.py

@@ -0,0 +1,6 @@
+import os
+from elasticsearch_dsl.connections import connections
+
+
+def get_connection():
+    return connections.create_connection(hosts=[os.getenv('ELASTICSEARCH_URL', '127.0.0.1:9200')])

utils/response.py → app/utils/response.py

@@ -1,5 +1,5 @@
-from models.product import Product
-from models.request import SearchBase
+from app.models.product import Product
+from app.models.request import SearchBase
 
 
 def create_response(es_results, request: SearchBase):

docker-compose.yml

@@ -123,7 +123,6 @@ services:
       timeout: 10s
       retries: 120
 
-
   # elasticsearch browser
   elasticvue:
       image: cars10/elasticvue
@@ -133,6 +132,14 @@ services:
       links:
           - es01
 
+  searchservice:
+    image: off_search_image
+    container_name: searchservice
+    environment:
+      - ELASTICSEARCH_URL=host.docker.internal:9200
+    ports:
+      - '8000:8000'
+
 volumes:
   certs:
     driver: local