Moved Config data model from SLC-CI-Setup to SLC-CI

exasol_script_languages_container_ci/cli/commands/run_ci.py

@@ -4,6 +4,7 @@
 
 from exasol_script_languages_container_ci.cli.cli import cli
 from exasol_script_languages_container_ci.lib.ci import ci
+from exasol_script_languages_container_ci.lib.config.config_data_model import Config
 from exasol_script_languages_container_ci.lib.git_access import GitAccess
 
 
@@ -35,12 +36,13 @@ def run_ci(ctx: click.Context,
            commit_sha: str,
            config_file: str):
     logging.basicConfig(level=logging.INFO)
-    ci(flavor,
-       branch_name,
-       docker_user,
-       docker_password,
-       docker_build_repository,
-       docker_release_repository,
-       commit_sha,
-       config_file,
-       GitAccess())
+    build_config = Config.parse_file(config_file)
+    ci(flavor=flavor,
+       branch_name=branch_name,
+       docker_user=docker_user,
+       docker_password=docker_password,
+       docker_build_repository=docker_build_repository,
+       docker_release_repository=docker_release_repository,
+       commit_sha=commit_sha,
+       build_config=build_config,
+       git_access=GitAccess())

exasol_script_languages_container_ci/cli/commands/run_release.py

@@ -5,6 +5,7 @@
 
 from exasol_script_languages_container_ci.cli.cli import cli
 from exasol_script_languages_container_ci.lib.asset_uploader import AssetUploader
+from exasol_script_languages_container_ci.lib.config.config_data_model import Config
 from exasol_script_languages_container_ci.lib.github_release_asset_uploader import GithubReleaseAssetUploader
 from exasol_script_languages_container_ci.lib.release import release
 from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader
@@ -42,11 +43,12 @@ def run_release(ctx: click.Context,
     github_release_asset_uploader = GithubReleaseAssetUploader(os.getenv("GITHUB_TOKEN"))
     asset_uploader = AssetUploader(release_asset_uploader=github_release_asset_uploader)
     release_uploader = ReleaseUploader(asset_uploader=asset_uploader)
+    build_config = Config.parse_file(config_file)
     release(flavor=flavor,
             docker_user=docker_user,
             docker_password=docker_password,
             docker_release_repository=docker_release_repository,
-            config_file=config_file,
+            build_config=build_config,
             source_repo_url=source_repo_url,
             release_id=release_id,
             release_uploader=release_uploader,

exasol_script_languages_container_ci/lib/ci.py

@@ -3,16 +3,15 @@
 from pathlib import Path
 from typing import Set
 
-import click
+from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY
 from exasol_integration_test_docker_environment.lib.base import luigi_log_config
-from exasol_integration_test_docker_environment.lib.config import build_config
 
 from exasol_script_languages_container_ci.lib.branch_config import BranchConfig
-from exasol_script_languages_container_ci.lib.common import get_config
 from exasol_script_languages_container_ci.lib.ci_build import CIBuild
 from exasol_script_languages_container_ci.lib.ci_push import CIPush
 from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan
 from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest
+from exasol_script_languages_container_ci.lib.config.config_data_model import Config
 from exasol_script_languages_container_ci.lib.git_access import GitAccess
 
 
@@ -26,16 +25,15 @@ def get_all_affected_files(git_access: GitAccess, base_branch: str) -> Set[str]:
     return changed_files
 
 
-def check_if_need_to_build(branch_name: str, config_file: str, flavor: str, git_access: GitAccess):
+def check_if_need_to_build(branch_name: str, config: Config, flavor: str, git_access: GitAccess):
     if BranchConfig.build_always(branch_name):
         return True
     if "[rebuild]" in git_access.get_last_commit_message():
         return True
-    with get_config(config_file) as config:
-        affected_files = list(get_all_affected_files(git_access, config["base_branch"]))
-        logging.debug(f"check_if_need_to_build: Found files of last commits: {affected_files}")
-        for ignore_path in config["build_ignore"]["ignored_paths"]:
-            affected_files = list(filter(lambda file: not file.startswith(ignore_path), affected_files))
+    affected_files = list(get_all_affected_files(git_access, config.build.base_branch))
+    logging.debug(f"check_if_need_to_build: Found files of last commits: {affected_files}")
+    for ignore_path in config.build.ignore.paths:
+        affected_files = list(filter(lambda file: not file.startswith(ignore_path), affected_files))
 
     if len(affected_files) > 0:
         # Now filter out also other flavor folders
@@ -53,7 +51,7 @@ def ci(flavor: str,
        docker_build_repository: str,
        docker_release_repository: str,
        commit_sha: str,
-       config_file: str,
+       build_config: Config,
        git_access: GitAccess,
        ci_build: CIBuild = CIBuild(),
        ci_execute_tests: CIExecuteTest = CIExecuteTest(),
@@ -71,9 +69,9 @@ def ci(flavor: str,
     flavor_path = (f"flavors/{flavor}",)
     test_container_folder = "test_container"
     rebuild = BranchConfig.rebuild(branch_name)
-    needs_to_build = check_if_need_to_build(branch_name, config_file, flavor, git_access)
+    needs_to_build = check_if_need_to_build(branch_name, build_config, flavor, git_access)
     if needs_to_build:
-        log_path = Path(build_config.DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
+        log_path = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
         os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] = f"{log_path.absolute()}"
 
         ci_build.build(flavor_path=flavor_path,

exasol_script_languages_container_ci/lib/config/data_model_generator.py

@@ -0,0 +1,51 @@
+import json
+import logging
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from datamodel_code_generator import generate, InputFileType
+
+from exasol_script_languages_container_ci.lib.render_template import render_template
+
+logger = logging.getLogger(__name__)
+
+CONFIG_DATA_MODEL_FILE_NAME = "config_data_model.py"
+
+
+def config_data_model_default_output_file() -> Path:
+    return Path(__file__).parent / CONFIG_DATA_MODEL_FILE_NAME
+
+
+def generate_config_data_model(output_file: Path) -> Path:
+    schema_dict = json.loads(render_template("config_schema.json"))
+    schema_json = json.dumps(schema_dict)
+    with TemporaryDirectory() as directory:
+        temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME
+        generate(schema_json, input_file_type=InputFileType.JsonSchema, output=temp_output_file,
+                 class_name="Config", apply_default_values_for_required_fields=True)
+        with temp_output_file.open("rt") as temp_output_file_handle:
+            with output_file.open("wt") as output_file_handle:
+                lines = (line for line in temp_output_file_handle)
+                lines = filter(lambda line: "#   timestamp: " not in line, lines)
+                for line in lines:
+                    output_file_handle.write(line)
+    return output_file
+
+
+def regenerate_config_data_model(output_file: Path):
+    with TemporaryDirectory() as directory:
+        temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME
+        generate_config_data_model(temp_output_file)
+        with temp_output_file.open("rt") as file:
+            new_model = file.read()
+        if output_file.is_file():
+            with output_file.open("rt") as file:
+                old_model = file.read()
+            if new_model != old_model:
+                logger.warning(f"Regenerating config pydantic model to {output_file}")
+                with output_file.open("wt") as file:
+                    file.write(new_model)
+        else:
+            logger.info(f"Generating config pydantic model to new file {output_file}")
+            with output_file.open("wt") as file:
+                file.write(new_model)

exasol_script_languages_container_ci/lib/release.py

@@ -2,21 +2,22 @@
 import os
 from pathlib import Path
 
+from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY
 from exasol_integration_test_docker_environment.lib.base import luigi_log_config
-from exasol_integration_test_docker_environment.lib.config import build_config
 
 from exasol_script_languages_container_ci.lib.ci_build import CIBuild
 from exasol_script_languages_container_ci.lib.ci_push import CIPush
 from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan
 from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest
+from exasol_script_languages_container_ci.lib.config.config_data_model import Config
 from exasol_script_languages_container_ci.lib.release_uploader import ReleaseUploader
 
 
 def release(flavor: str,
             docker_user: str,
             docker_password: str,
             docker_release_repository: str,
-            config_file: str,
+            build_config: Config,
             source_repo_url: str,
             release_id: int,
             is_dry_run: bool,
@@ -36,7 +37,7 @@ def release(flavor: str,
 
     flavor_path = (f"flavors/{flavor}",)
     test_container_folder = "test_container"
-    log_path = Path(build_config.DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
+    log_path = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
     os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] = f"{log_path.absolute()}"
 
     ci_build.build(flavor_path=flavor_path, rebuild=True,

exasol_script_languages_container_ci/lib/render_template.py

@@ -0,0 +1,8 @@
+import jinja2
+
+
+def render_template(template: str, **kwargs):
+    env = jinja2.Environment(loader=jinja2.PackageLoader("exasol_script_languages_container_ci"),
+                             autoescape=jinja2.select_autoescape(), keep_trailing_newline=True)
+    t = env.get_template(template)
+    return t.render(**kwargs)

exasol_script_languages_container_ci/templates/config_schema.json

@@ -0,0 +1,50 @@
+{
+  "type": "object",
+  "required": [
+    "build",
+    "release"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "build": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "ignore",
+        "base_branch"
+      ],
+      "properties": {
+        "ignore": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "paths"
+          ],
+          "properties": {
+            "paths": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        },
+        "base_branch": {
+          "type": "string"
+        }
+      }
+    },
+    "release": {
+      "type": "object",
+      "required": [
+        "timeout_in_minutes"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "timeout_in_minutes": {
+          "type": "integer"
+        }
+      }
+    }
+  }
+}