#11: Improved control of CI builds (#18)

fixes #11

doc/changes/changelog.md

@@ -1,5 +1,6 @@
 # Changes
 
+* [0.4.0](changes_0.4.0.md)
 * [0.3.0](changes_0.3.0.md)
 * [0.2.0](changes_0.2.0.md)
 * [0.1.0](changes_0.1.0.md)

doc/changes/changes_0.4.0.md

@@ -0,0 +1,23 @@
+# script-languages-container-ci 0.4.0, released 2022-06-14
+
+Code name: Changed git history analysis
+
+## Summary
+
+This release checks for the word '[rebuild]' in the commit message and the ci checks all new commits to evaluate the need to build and test.
+
+## Bug Fixes
+
+n/a
+
+## Features / Enhancements
+
+ -  #11: Improved control of CI builds
+
+## Documentation
+
+n/a
+
+## Refactoring
+
+n/a

exasol_script_languages_container_ci/lib/branch_config.py

@@ -4,29 +4,24 @@
 
 class BuildSteps(Enum):
     BUILD_ALL_ALWAYS = auto()
-    TEST_ALWAYS = auto()
     REBUILD = auto()
     PUSH_TO_DOCKER_RELEASE_REPO = auto()
 
 
 class BranchConfig(Enum):
-    DEVELOP = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True, BuildSteps.TEST_ALWAYS: True,
+    DEVELOP = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True,
                BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False}
-    MAIN = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True, BuildSteps.TEST_ALWAYS: True,
+    MAIN = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True,
               BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: True}
-    REBUILD = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True, BuildSteps.TEST_ALWAYS: True,
+    REBUILD = {BuildSteps.BUILD_ALL_ALWAYS: True, BuildSteps.REBUILD: True,
                BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False}
-    OTHER = {BuildSteps.BUILD_ALL_ALWAYS: False, BuildSteps.REBUILD: False, BuildSteps.TEST_ALWAYS: False,
+    OTHER = {BuildSteps.BUILD_ALL_ALWAYS: False, BuildSteps.REBUILD: False,
              BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False}
 
     @staticmethod
     def build_always(branch_name: str) -> bool:
         return get_branch_config(branch_name).value[BuildSteps.BUILD_ALL_ALWAYS]
 
-    @staticmethod
-    def test_always(branch_name) -> bool:
-        return get_branch_config(branch_name).value[BuildSteps.TEST_ALWAYS]
-
     @staticmethod
     def rebuild(branch_name) -> bool:
         return get_branch_config(branch_name).value[BuildSteps.REBUILD]

exasol_script_languages_container_ci/lib/ci.py

@@ -1,8 +1,7 @@
 import logging
 import os
-import re
 from pathlib import Path
-from typing import Tuple
+from typing import Set
 
 import click
 from exasol_integration_test_docker_environment.lib.base import luigi_log_config
@@ -13,14 +12,28 @@
 from exasol_script_languages_container_ci.lib.ci_build import ci_build
 from exasol_script_languages_container_ci.lib.ci_push import ci_push
 from exasol_script_languages_container_ci.lib.ci_security_scan import ci_security_scan
-from exasol_script_languages_container_ci.lib.ci_test import ci_test
+from exasol_script_languages_container_ci.lib.ci_test import execute_tests
 from exasol_script_languages_container_ci.lib.git_access import GitAccess
 
 
+def get_all_affected_files(git_access: GitAccess, base_branch: str) -> Set[str]:
+    base_last_commit_sha = git_access.get_head_commit_sha_of_branch(base_branch)
+    changed_files = set()
+    for commit in git_access.get_last_commits():
+        if commit == base_last_commit_sha:
+            break
+        changed_files.update(git_access.get_files_of_commit(commit))
+    return changed_files
+
+
 def check_if_need_to_build(branch_name: str, config_file: str, flavor: str, git_access: GitAccess):
-    affected_files = list(git_access.get_files_of_last_commit())
-    logging.debug(f"check_if_need_to_build: Found files of last commit: {affected_files}")
+    if BranchConfig.build_always(branch_name):
+        return True
+    if "[rebuild]" in git_access.get_last_commit_message():
+        return True
     with get_config(config_file) as config:
+        affected_files = list(get_all_affected_files(git_access, config["base_branch"]))
+        logging.debug(f"check_if_need_to_build: Found files of last commits: {affected_files}")
         for ignore_path in config["build_ignore"]["ignored_paths"]:
             affected_files = list(filter(lambda file: not file.startswith(ignore_path), affected_files))
 
@@ -30,7 +43,7 @@ def check_if_need_to_build(branch_name: str, config_file: str, flavor: str, git_
         affected_files = list(filter(lambda file: not file.startswith("flavors") or file.startswith(this_flavor_path),
                                      affected_files))
     logging.debug(f"check_if_need_to_build: filtered files: {affected_files}")
-    return len(affected_files) > 0 or BranchConfig.build_always(branch_name)
+    return len(affected_files) > 0
 
 
 def ci(ctx: click.Context,
@@ -62,8 +75,7 @@ def ci(ctx: click.Context,
         ci_build(ctx, flavor_path=flavor_path, rebuild=rebuild, build_docker_repository=docker_build_repository,
                  commit_sha=commit_sha,
                  docker_user=docker_user, docker_password=docker_password)
-        ci_test(ctx, flavor_path=flavor_path, branch_name=branch_name, git_access=git_access,
-                docker_user=docker_user, docker_password=docker_password)
+        execute_tests(ctx, flavor_path=flavor_path, docker_user=docker_user, docker_password=docker_password)
         ci_security_scan(ctx, flavor_path=flavor_path)
         ci_push(ctx, flavor_path=flavor_path,
                 target_docker_repository=docker_build_repository, target_docker_tag_prefix=commit_sha,

exasol_script_languages_container_ci/lib/ci_test.py

@@ -4,13 +4,7 @@
 import click
 from exasol_script_languages_container_tool.cli.commands import run_db_test
 
-from exasol_script_languages_container_ci.lib.branch_config import BranchConfig
 from exasol_script_languages_container_ci.lib.common import print_docker_images
-from exasol_script_languages_container_ci.lib.git_access import GitAccess
-
-
-def _need_to_run_tests(branch_name: str, git_access: GitAccess):
-    return BranchConfig.test_always(branch_name) or "[skip tests]" not in git_access.get_last_commit_message()
 
 
 def execute_tests(ctx: click.Context, flavor_path: Tuple[str, ...], docker_user: str, docker_password: str):
@@ -25,11 +19,3 @@ def execute_tests(ctx: click.Context, flavor_path: Tuple[str, ...], docker_user:
                test_folder=("test/linker_namespace_sanity",), release_goal=("base_test_build_run",),
                source_docker_username=docker_user, source_docker_password=docker_password)
     print_docker_images(logging.info)
-
-
-def ci_test(ctx: click.Context, flavor_path: Tuple[str, ...], branch_name: str, git_access: GitAccess,
-            docker_user: str, docker_password: str):
-    if _need_to_run_tests(branch_name, git_access):
-        execute_tests(ctx, flavor_path, docker_user, docker_password)
-    else:
-        logging.warning("Skipping tests.")

exasol_script_languages_container_ci/lib/git_access.py

@@ -12,11 +12,33 @@ def get_last_commit_message(self):
         """
         return Repo().head.commit.message
 
-    def get_files_of_last_commit(self) -> Iterable[str]:
+    def get_head_commit_sha_of_branch(self, branch_name) -> str:
         """
-        Returns the files of the last commit of the repo in the cwd.
+        Returns the last commit sha of given branch.
+        :raise: ValueError: if the refs with label 'branch_name' does not exists or is not unique.
         """
         repo = Repo()
-        commit = repo.head.commit
-        return commit.stats.files.keys()
+        branch = [b for b in repo.refs if b.name == branch_name]
+        if len(branch) == 0:
+            ex_msg = f"Branch '{branch_name}' does not exist."
+            raise ValueError(ex_msg)
+        elif len(branch) > 1:
+            ex_msg = f"Branch '{branch_name}' found more than once."
+            raise ValueError(ex_msg)
+        return str(branch[0].commit)
+
+    def get_last_commits(self) -> Iterable[str]:
+        """
+        Returns all commit-sha's of the current branch of the repo in the cwd.
+        """
+        repo = Repo()
+        for c in repo.iter_commits(repo.head):
+            yield str(c)
+
+    def get_files_of_commit(self, commit_sha) -> Iterable[str]:
+        """
+        Returns the files of the specific commits of the repo in the cwd.
+        """
+        repo = Repo()
+        return repo.commit(commit_sha).stats.files.keys()
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-script-languages-container-ci"
-version = "0.3.0"
+version = "0.4.0"
 description = "Implements CI builds for script-language-container."
 
 license = "MIT"

setup.py

@@ -21,7 +21,7 @@
 
 setup_kwargs = {
     'name': 'exasol-script-languages-container-ci',
-    'version': '0.3.0',
+    'version': '0.4.0',
     'description': 'Implements CI builds for script-language-container.',
     'long_description': None,
     'author': 'Thomas Uebensee',

test/fixtures.py

@@ -30,7 +30,7 @@ def click_stub():
 @pytest.fixture
 def config_file(tmp_path_factory):
     config_file_path = tmp_path_factory.mktemp("config") / "build_config.json"
-    config = {"build_ignore": {"ignored_paths": ["doc"]}}
+    config = {"build_ignore": {"ignored_paths": ["doc"]}, "base_branch": "master"}
     with open(config_file_path, "w") as f:
         json.dump(config, f)
     return config_file_path
@@ -52,6 +52,8 @@ def git_access_mock():
     Return an object which mocks the git access class. The mock object returns some default values useful for the tests.
     """
     git_access_mock = MagicMock()
-    git_access_mock.get_files_of_last_commit.return_value = ["src/udfclient.cpp"]
+    git_access_mock.get_last_commits.return_value = ["456", "123"]
+    git_access_mock.get_head_commit_sha_of_branch.return_value = "123"
+    git_access_mock.get_files_of_commit.return_value = ["src/udfclient.cpp"]
     git_access_mock.get_last_commit_message.return_value = "last commit"
     return git_access_mock