From e575b28da912147c5b806abab40a0c92329e2eb7 Mon Sep 17 00:00:00 2001 From: Roy Moore Date: Fri, 28 Jun 2024 11:34:41 +0300 Subject: [PATCH] feat(core): Added Generic module (#612) As part of the effort described, detailed and presented on /~https://github.com/testcontainers/testcontainers-python/pull/559 This is the third PR (out of 4) that should provide all the groundwork to support containers running a server. As discussed on #595 this PR aims to refactor the `ServerContainer` under a new dedicated module called "generic". ![image](/~https://github.com/testcontainers/testcontainers-python/assets/7189138/b7a3395b-ce3c-40ef-8baa-dfa3eff1b056) The idea is that this module could include multiple generic implementations such as ```server.py``` with the proper documentation and examples to allow users simpler usage and QOL. This PR adds the original FastAPI implementation as a simple doc example, I think this aligns better following #595 Next in line is ```feat(core): Added AWS Lambda module``` Based on the work done on /~https://github.com/testcontainers/testcontainers-python/pull/585 and #595 Expended from issue /~https://github.com/testcontainers/testcontainers-python/issues/83 --- Please note an extra commit is included to simulate the relations when importing between and with other modules. --- core/README.rst | 26 +++--- core/testcontainers/core/generic.py | 71 +--------------- index.rst | 11 +++ modules/generic/README.rst | 20 +++++ .../testcontainers/generic/__init__.py | 1 + .../generic/testcontainers/generic/server.py | 80 +++++++++++++++++++ modules/generic/tests/conftest.py | 22 +++++ .../generic/tests/samples/fastapi/Dockerfile | 11 +++ .../tests/samples/fastapi/app/__init__.py | 0 .../generic/tests/samples/fastapi/app/main.py | 8 ++ .../tests/samples}/python_server/Dockerfile | 0 .../generic/tests/test_generic.py | 14 +++- modules/testmoduleimport/README.rst | 2 + .../testmoduleimport/__init__.py | 1 + .../testmoduleimport/new_sub_module.py | 27 +++++++ .../testmoduleimport/tests/test_mock_one.py | 15 ++++ poetry.lock | 5 +- pyproject.toml | 7 +- 18 files changed, 232 insertions(+), 89 deletions(-) create mode 100644 modules/generic/README.rst create mode 100644 modules/generic/testcontainers/generic/__init__.py create mode 100644 modules/generic/testcontainers/generic/server.py create mode 100644 modules/generic/tests/conftest.py create mode 100644 modules/generic/tests/samples/fastapi/Dockerfile create mode 100644 modules/generic/tests/samples/fastapi/app/__init__.py create mode 100644 modules/generic/tests/samples/fastapi/app/main.py rename {core/tests/image_fixtures => modules/generic/tests/samples}/python_server/Dockerfile (100%) rename core/tests/test_generics.py => modules/generic/tests/test_generic.py (74%) create mode 100644 modules/testmoduleimport/README.rst create mode 100644 modules/testmoduleimport/testcontainers/testmoduleimport/__init__.py create mode 100644 modules/testmoduleimport/testcontainers/testmoduleimport/new_sub_module.py create mode 100644 modules/testmoduleimport/tests/test_mock_one.py diff --git a/core/README.rst b/core/README.rst index 8479efac..8cc9a278 100644 --- a/core/README.rst +++ b/core/README.rst @@ -5,7 +5,18 @@ Testcontainers Core .. autoclass:: testcontainers.core.container.DockerContainer -Using `DockerContainer` and `DockerImage` directly: +.. autoclass:: testcontainers.core.image.DockerImage + +.. autoclass:: testcontainers.core.generic.DbContainer + +.. raw:: html + +
+ +Examples +-------- + +Using `DockerContainer` and `DockerImage` to create a container: .. doctest:: @@ -17,14 +28,5 @@ Using `DockerContainer` and `DockerImage` directly: ... with DockerContainer(str(image)) as container: ... delay = wait_for_logs(container, "Test Sample Image") ---- - -.. autoclass:: testcontainers.core.image.DockerImage - ---- - -.. autoclass:: testcontainers.core.generic.ServerContainer - ---- - -.. autoclass:: testcontainers.core.generic.DbContainer +The `DockerImage` class is used to build the image from the specified path and tag. +The `DockerContainer` class is then used to create a container from the image. diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index 11456a51..b2cd3010 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -10,14 +10,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from typing import Optional, Union -from urllib.error import HTTPError +from typing import Optional from urllib.parse import quote -from urllib.request import urlopen from testcontainers.core.container import DockerContainer from testcontainers.core.exceptions import ContainerStartException -from testcontainers.core.image import DockerImage from testcontainers.core.utils import raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_container_is_ready @@ -84,69 +81,3 @@ def _configure(self) -> None: def _transfer_seed(self) -> None: pass - - -class ServerContainer(DockerContainer): - """ - **DEPRECATED - will be moved from core to a module (stay tuned for a final/stable import location)** - - Container for a generic server that is based on a custom image. - - Example: - - .. doctest:: - - >>> import httpx - >>> from testcontainers.core.generic import ServerContainer - >>> from testcontainers.core.waiting_utils import wait_for_logs - >>> from testcontainers.core.image import DockerImage - - >>> with DockerImage(path="./core/tests/image_fixtures/python_server", tag="test-srv:latest") as image: - ... with ServerContainer(port=9000, image=image) as srv: - ... url = srv._create_connection_url() - ... response = httpx.get(f"{url}", timeout=5) - ... assert response.status_code == 200, "Response status code is not 200" - ... delay = wait_for_logs(srv, "GET / HTTP/1.1") - - - :param path: Path to the Dockerfile to build the image - :param tag: Tag for the image to be built (default: None) - """ - - def __init__(self, port: int, image: Union[str, DockerImage]) -> None: - super().__init__(str(image)) - self.internal_port = port - self.with_exposed_ports(self.internal_port) - - @wait_container_is_ready(HTTPError) - def _connect(self) -> None: - # noinspection HttpUrlsUsage - url = self._create_connection_url() - try: - with urlopen(url) as r: - assert b"" in r.read() - except HTTPError as e: - # 404 is expected, as the server may not have the specific endpoint we are looking for - if e.code == 404: - pass - else: - raise - - def get_api_url(self) -> str: - raise NotImplementedError - - def _create_connection_url(self) -> str: - if self._container is None: - raise ContainerStartException("container has not been started") - host = self.get_container_host_ip() - exposed_port = self.get_exposed_port(self.internal_port) - url = f"http://{host}:{exposed_port}" - return url - - def start(self) -> "ServerContainer": - super().start() - self._connect() - return self - - def stop(self, force=True, delete_volume=True) -> None: - super().stop(force, delete_volume) diff --git a/index.rst b/index.rst index af314283..ead699b2 100644 --- a/index.rst +++ b/index.rst @@ -70,6 +70,17 @@ Please note, that community modules are supported on a best-effort basis and bre Therefore, only the package core is strictly following SemVer. If your workflow is broken by a minor update, please look at the changelogs for guidance. +Custom Containers +----------------- + +Crafting containers that are based on custom images is supported by the `core` module. Please check the `core documentation `_ for more information. + +This allows you to create containers from images that are not part of the modules provided by testcontainers-python. + +For common use cases, you can also use the generic containers provided by the `testcontainers-generic` module. Please check the `generic documentation `_ for more information. +(example: `ServerContainer` for running a FastAPI server) + + Docker in Docker (DinD) ----------------------- diff --git a/modules/generic/README.rst b/modules/generic/README.rst new file mode 100644 index 00000000..7e12da70 --- /dev/null +++ b/modules/generic/README.rst @@ -0,0 +1,20 @@ +:code:`testcontainers-generic` is a set of generic containers modules that can be used to creat containers. + +.. autoclass:: testcontainers.generic.ServerContainer +.. title:: testcontainers.generic.ServerContainer + +FastAPI container that is using :code:`ServerContainer` + +.. doctest:: + + >>> from testcontainers.generic import ServerContainer + >>> from testcontainers.core.waiting_utils import wait_for_logs + + >>> with DockerImage(path="./modules/generic/tests/samples/fastapi", tag="fastapi-test:latest") as image: + ... with ServerContainer(port=80, image=image) as fastapi_server: + ... delay = wait_for_logs(fastapi_server, "Uvicorn running on http://0.0.0.0:80") + ... fastapi_server.get_api_url = lambda: fastapi_server._create_connection_url() + "/api/v1/" + ... client = fastapi_server.get_client() + ... response = client.get("/") + ... assert response.status_code == 200 + ... assert response.json() == {"Status": "Working"} diff --git a/modules/generic/testcontainers/generic/__init__.py b/modules/generic/testcontainers/generic/__init__.py new file mode 100644 index 00000000..f239a80c --- /dev/null +++ b/modules/generic/testcontainers/generic/__init__.py @@ -0,0 +1 @@ +from .server import ServerContainer # noqa: F401 diff --git a/modules/generic/testcontainers/generic/server.py b/modules/generic/testcontainers/generic/server.py new file mode 100644 index 00000000..03a54677 --- /dev/null +++ b/modules/generic/testcontainers/generic/server.py @@ -0,0 +1,80 @@ +from typing import Union +from urllib.error import HTTPError +from urllib.request import urlopen + +import httpx + +from testcontainers.core.container import DockerContainer +from testcontainers.core.exceptions import ContainerStartException +from testcontainers.core.image import DockerImage +from testcontainers.core.waiting_utils import wait_container_is_ready + + +class ServerContainer(DockerContainer): + """ + Container for a generic server that is based on a custom image. + + Example: + + .. doctest:: + + >>> import httpx + >>> from testcontainers.generic import ServerContainer + >>> from testcontainers.core.waiting_utils import wait_for_logs + >>> from testcontainers.core.image import DockerImage + + >>> with DockerImage(path="./modules/generic/tests/samples/python_server", tag="test-srv:latest") as image: + ... with ServerContainer(port=9000, image=image) as srv: + ... url = srv._create_connection_url() + ... response = httpx.get(f"{url}", timeout=5) + ... assert response.status_code == 200, "Response status code is not 200" + ... delay = wait_for_logs(srv, "GET / HTTP/1.1") + + + :param path: Path to the Dockerfile to build the image + :param tag: Tag for the image to be built (default: None) + """ + + def __init__(self, port: int, image: Union[str, DockerImage]) -> None: + super().__init__(str(image)) + self.internal_port = port + self.with_exposed_ports(self.internal_port) + + @wait_container_is_ready(HTTPError) + def _connect(self) -> None: + # noinspection HttpUrlsUsage + url = self._create_connection_url() + try: + with urlopen(url) as r: + assert b"" in r.read() + except HTTPError as e: + # 404 is expected, as the server may not have the specific endpoint we are looking for + if e.code == 404: + pass + else: + raise + + def get_api_url(self) -> str: + raise NotImplementedError + + def _create_connection_url(self) -> str: + if self._container is None: + raise ContainerStartException("container has not been started") + host = self.get_container_host_ip() + exposed_port = self.get_exposed_port(self.internal_port) + url = f"http://{host}:{exposed_port}" + return url + + def start(self) -> "ServerContainer": + super().start() + self._connect() + return self + + def stop(self, force=True, delete_volume=True) -> None: + super().stop(force, delete_volume) + + def get_client(self) -> httpx.Client: + return httpx.Client(base_url=self.get_api_url()) + + def get_stdout(self) -> str: + return self.get_logs()[0].decode("utf-8") diff --git a/modules/generic/tests/conftest.py b/modules/generic/tests/conftest.py new file mode 100644 index 00000000..4f69565f --- /dev/null +++ b/modules/generic/tests/conftest.py @@ -0,0 +1,22 @@ +import pytest +from typing import Callable +from testcontainers.core.container import DockerClient + + +@pytest.fixture +def check_for_image() -> Callable[[str, bool], None]: + """Warp the check_for_image function in a fixture""" + + def _check_for_image(image_short_id: str, cleaned: bool) -> None: + """ + Validates if the image is present or not. + + :param image_short_id: The short id of the image + :param cleaned: True if the image should not be present, False otherwise + """ + client = DockerClient() + images = client.client.images.list() + found = any(image.short_id.endswith(image_short_id) for image in images) + assert found is not cleaned, f'Image {image_short_id} was {"found" if cleaned else "not found"}' + + return _check_for_image diff --git a/modules/generic/tests/samples/fastapi/Dockerfile b/modules/generic/tests/samples/fastapi/Dockerfile new file mode 100644 index 00000000..f56288cd --- /dev/null +++ b/modules/generic/tests/samples/fastapi/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.9 + +WORKDIR /app + +RUN pip install fastapi + +COPY ./app /app + +EXPOSE 80 + +CMD ["fastapi", "run", "main.py", "--port", "80"] diff --git a/modules/generic/tests/samples/fastapi/app/__init__.py b/modules/generic/tests/samples/fastapi/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/generic/tests/samples/fastapi/app/main.py b/modules/generic/tests/samples/fastapi/app/main.py new file mode 100644 index 00000000..f96073d9 --- /dev/null +++ b/modules/generic/tests/samples/fastapi/app/main.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/api/v1/") +def read_root(): + return {"Status": "Working"} diff --git a/core/tests/image_fixtures/python_server/Dockerfile b/modules/generic/tests/samples/python_server/Dockerfile similarity index 100% rename from core/tests/image_fixtures/python_server/Dockerfile rename to modules/generic/tests/samples/python_server/Dockerfile diff --git a/core/tests/test_generics.py b/modules/generic/tests/test_generic.py similarity index 74% rename from core/tests/test_generics.py rename to modules/generic/tests/test_generic.py index 340ac665..5943b4a4 100644 --- a/core/tests/test_generics.py +++ b/modules/generic/tests/test_generic.py @@ -7,17 +7,17 @@ from testcontainers.core.waiting_utils import wait_for_logs from testcontainers.core.image import DockerImage -from testcontainers.core.generic import ServerContainer +from testcontainers.generic import ServerContainer TEST_DIR = Path(__file__).parent @pytest.mark.parametrize("test_image_cleanup", [True, False]) @pytest.mark.parametrize("test_image_tag", [None, "custom-image:test"]) -def test_srv_container(test_image_tag: Optional[str], test_image_cleanup: bool, check_for_image, port=9000): +def test_server_container(test_image_tag: Optional[str], test_image_cleanup: bool, check_for_image, port=9000): with ( DockerImage( - path=TEST_DIR / "image_fixtures/python_server", + path=TEST_DIR / "samples/python_server", tag=test_image_tag, clean_up=test_image_cleanup, # @@ -37,8 +37,14 @@ def test_srv_container(test_image_tag: Optional[str], test_image_cleanup: bool, check_for_image(image_short_id, test_image_cleanup) +def test_server_container_no_port(): + with pytest.raises(TypeError): + with ServerContainer(path="./modules/generic/tests/samples/python_server", tag="test-srv:latest"): + pass + + def test_like_doctest(): - with DockerImage(path=TEST_DIR / "image_fixtures/python_server", tag="test-srv:latest") as image: + with DockerImage(path=TEST_DIR / "samples/python_server", tag="test-srv:latest") as image: with ServerContainer(port=9000, image=image) as srv: url = srv._create_connection_url() response = get(f"{url}", timeout=5) diff --git a/modules/testmoduleimport/README.rst b/modules/testmoduleimport/README.rst new file mode 100644 index 00000000..ae5d5708 --- /dev/null +++ b/modules/testmoduleimport/README.rst @@ -0,0 +1,2 @@ +.. autoclass:: testcontainers.testmoduleimport.NewSubModuleContainer +.. title:: testcontainers.testmoduleimport.NewSubModuleContainer diff --git a/modules/testmoduleimport/testcontainers/testmoduleimport/__init__.py b/modules/testmoduleimport/testcontainers/testmoduleimport/__init__.py new file mode 100644 index 00000000..74074699 --- /dev/null +++ b/modules/testmoduleimport/testcontainers/testmoduleimport/__init__.py @@ -0,0 +1 @@ +from .new_sub_module import NewSubModuleContainer # noqa: F401 diff --git a/modules/testmoduleimport/testcontainers/testmoduleimport/new_sub_module.py b/modules/testmoduleimport/testcontainers/testmoduleimport/new_sub_module.py new file mode 100644 index 00000000..f45796f7 --- /dev/null +++ b/modules/testmoduleimport/testcontainers/testmoduleimport/new_sub_module.py @@ -0,0 +1,27 @@ +from testcontainers.generic.server import ServerContainer + + +class NewSubModuleContainer(ServerContainer): + """ + This class is a mock container for testing purposes. It is used to test importing from other modules. + + .. doctest:: + + >>> import httpx + >>> from testcontainers.core.image import DockerImage + >>> from testcontainers.testmoduleimport import NewSubModuleContainer + + >>> with DockerImage(path="./modules/generic/tests/samples/python_server", tag="test-mod:latest") as image: + ... with NewSubModuleContainer(port=9000, image=image) as srv: + ... url = srv._create_connection_url() + ... response = httpx.get(f"{url}", timeout=5) + ... assert response.status_code == 200, "Response status code is not 200" + ... assert srv.print_mock() == "NewSubModuleContainer" + + """ + + def __init__(self, port: int, image: str) -> None: + super().__init__(port, image) + + def print_mock(self) -> str: + return "NewSubModuleContainer" diff --git a/modules/testmoduleimport/tests/test_mock_one.py b/modules/testmoduleimport/tests/test_mock_one.py new file mode 100644 index 00000000..85ac6c31 --- /dev/null +++ b/modules/testmoduleimport/tests/test_mock_one.py @@ -0,0 +1,15 @@ +import httpx + +from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.core.image import DockerImage +from testcontainers.testmoduleimport import NewSubModuleContainer + + +def test_like_doctest(): + with DockerImage(path="./modules/generic/tests/samples/python_server", tag="test-srv:latest") as image: + with NewSubModuleContainer(port=9000, image=image) as srv: + assert srv.print_mock() == "NewSubModuleContainer" + url = srv._create_connection_url() + response = httpx.get(f"{url}", timeout=5) + assert response.status_code == 200, "Response status code is not 200" + _ = wait_for_logs(srv, "GET / HTTP/1.1") diff --git a/poetry.lock b/poetry.lock index b2cb7e81..aa5fdc29 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1898,7 +1898,6 @@ python-versions = ">=3.7" files = [ {file = "milvus_lite-2.4.7-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:c828190118b104b05b8c8e0b5a4147811c86b54b8fb67bc2e726ad10fc0b544e"}, {file = "milvus_lite-2.4.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e1537633c39879714fb15082be56a4b97f74c905a6e98e302ec01320561081af"}, - {file = "milvus_lite-2.4.7-py3-none-manylinux2014_aarch64.whl", hash = "sha256:fcb909d38c83f21478ca9cb500c84264f988c69f62715ae9462e966767fb76dd"}, {file = "milvus_lite-2.4.7-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f016474d663045787dddf1c3aad13b7d8b61fd329220318f858184918143dcbf"}, ] @@ -4464,6 +4463,7 @@ chroma = ["chromadb-client"] clickhouse = ["clickhouse-driver"] cockroachdb = [] elasticsearch = [] +generic = ["httpx"] google = ["google-cloud-datastore", "google-cloud-pubsub"] influxdb = ["influxdb", "influxdb-client"] k3s = ["kubernetes", "pyyaml"] @@ -4490,10 +4490,11 @@ rabbitmq = ["pika"] redis = ["redis"] registry = ["bcrypt"] selenium = ["selenium"] +testmoduleimport = ["httpx"] vault = [] weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "6f7697a84a674802e30ceea61276d800b6b98224863a0c512138447d9b4af524" +content-hash = "e07f8edf8cefba872bbf48dcfa187163cefb00a60122daa62de8891b61fc55de" diff --git a/pyproject.toml b/pyproject.toml index cf9a3710..0b608895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ packages = [ { include = "testcontainers", from = "modules/clickhouse" }, { include = "testcontainers", from = "modules/cockroachdb" }, { include = "testcontainers", from = "modules/elasticsearch" }, + { include = "testcontainers", from = "modules/generic" }, + { include = "testcontainers", from = "modules/testmoduleimport"}, { include = "testcontainers", from = "modules/google" }, { include = "testcontainers", from = "modules/influxdb" }, { include = "testcontainers", from = "modules/k3s" }, @@ -61,7 +63,7 @@ packages = [ { include = "testcontainers", from = "modules/registry" }, { include = "testcontainers", from = "modules/selenium" }, { include = "testcontainers", from = "modules/vault" }, - { include = "testcontainers", from = "modules/weaviate" } + { include = "testcontainers", from = "modules/weaviate" }, ] [tool.poetry.urls] @@ -103,6 +105,7 @@ weaviate-client = { version = "^4.5.4", optional = true } chromadb-client = { version = "*", optional = true } qdrant-client = { version = "*", optional = true } bcrypt = { version = "*", optional = true } +httpx = { version = "*", optional = true } [tool.poetry.extras] arangodb = ["python-arango"] @@ -111,6 +114,8 @@ cassandra = [] clickhouse = ["clickhouse-driver"] cockroachdb = [] elasticsearch = [] +generic = ["httpx"] +testmoduleimport = ["httpx"] google = ["google-cloud-pubsub", "google-cloud-datastore"] influxdb = ["influxdb", "influxdb-client"] k3s = ["kubernetes", "pyyaml"]