diff --git a/.github/scripts/bake-metadata.py b/.github/scripts/bake-metadata.py index afda3c320b5..aabd2408bc6 100755 --- a/.github/scripts/bake-metadata.py +++ b/.github/scripts/bake-metadata.py @@ -86,6 +86,12 @@ def get_stable_tag(self, target: Target) -> str: version = self.get_stable_version() return self.tag(target, version) + def output_method(self): + return "registry" + + def get_output(self, target: Target) -> List[str]: + return ["type=registry"] + def get_tags(self, target: Target) -> List[str]: return [self.get_stable_tag(target)] @@ -98,10 +104,7 @@ def get_cache_from(self, target: Target) -> List[str]: @dataclass class PullRequestEvent(BaseEvent): - # the namespace is where images get pushed - # if the image is a fork, this can't be the target repo - namespace: str - + is_fork: bool pr_id: str pr_branch: str # the target branch name @@ -114,9 +117,6 @@ class PullRequestEvent(BaseEvent): # the target branch commit hash target_hash: str - def get_namespace(self): - return self.namespace - def version_string(self): return ( f"pr {self.pr_id} (" @@ -128,6 +128,16 @@ def get_stable_version(self) -> str: # edge/osrd-front:pr-42-HASH-nginx return f"pr-{self.pr_id}-{self.merge_hash}" + def output_method(self): + if not self.is_fork: + return super().output_method() + return "artifact" + + def get_output(self, target: Target) -> List[str]: + if not self.is_fork: + return super().get_output(target) + return [f"type=docker,dest=osrd-{target.name}.tar"] + def pr_tag(self, target: Target) -> str: # edge/osrd-front:pr-42-nginx # pr-42 (merge of XXXX into XXXX) return self.tag(target, f"pr-{self.pr_id}") @@ -136,11 +146,16 @@ def get_tags(self, target: Target) -> List[str]: return [*super().get_tags(target), self.pr_tag(target)] def get_cache_to(self, target: Target) -> List[str]: + if self.is_fork: + return [] return [registry_cache(self.pr_tag(target))] def get_cache_from(self, target: Target) -> List[str]: + target_branch_cache = registry_cache(self.tag(target, self.target_branch)) + if self.is_fork: + return [target_branch_cache] return [ - registry_cache(self.tag(target, self.target_branch)), + target_branch_cache, registry_cache(self.pr_tag(target)), ] @@ -261,17 +276,9 @@ def parse_event(context) -> Event: target_branch = context["base_ref"] orig_hash, target_hash = parse_merge_commit(commit_hash) repo_ctx = context["event"]["pull_request"]["head"]["repo"] - # if the repository is a fork, packages cannot be pushed to - # the upstream registry. Instead, we push to the fork's registry. - # The default namespace is suffixed with -edge, hence the special case - namespace = DEFAULT_EDGE_NAMESPACE - if repo_ctx["fork"]: - # multun/osrd - repo_name = repo_ctx["full_name"] - # ghcr wants lowercase org names - namespace = f"ghcr.io/{repo_name.lower()}" + is_fork = repo_ctx["fork"] return PullRequestEvent( - namespace=namespace, + is_fork=is_fork, pr_id=parse_pr_id(ref), pr_branch=context["head_ref"], target_branch=target_branch, @@ -286,7 +293,10 @@ def generate_bake_file(event, targets): bake_targets = {} for target in targets: # TODO: add labels - target_manifest = {"tags": event.get_tags(target)} + target_manifest = { + "tags": event.get_tags(target), + "output": event.get_output(target), + } if cache_to := event.get_cache_to(target): target_manifest["cache-to"] = cache_to if cache_from := event.get_cache_from(target): @@ -311,6 +321,7 @@ def main(): stable_tags[target.name] = event.get_stable_tag(target) print(f"stable_version={event.get_stable_version()}", file=f) print(f"stable_tags={json.dumps(stable_tags)}", file=f) + print(f"output_method={event.output_method()}", file=f) if __name__ == "__main__": diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd3986f580d..99c27134768 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,7 @@ jobs: outputs: stable_tags: ${{ steps.bake-metadata.outputs.stable_tags }} stable_version: ${{ steps.bake-metadata.outputs.stable_version }} + output_method: ${{ steps.bake-metadata.outputs.output_method }} steps: - name: Checkout uses: actions/checkout@v4 @@ -72,7 +73,7 @@ jobs: for i in $(seq 1 3); do echo "::group::Try $i" - if docker buildx bake $BAKEFILE $METADATA --push 2>&1 | tee docker-build.log; then + if docker buildx bake $BAKEFILE $METADATA 2>&1 | tee docker-build.log; then echo "::endgroup::" exit 0 fi @@ -91,6 +92,67 @@ jobs: echo "All retries failed, exiting." exit 1 + - name: Upload front-build artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: front-build + path: osrd-front-build.tar + - name: Upload core-build artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: core-build + path: osrd-core-build.tar + - name: Upload editoast-test artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: editoast-test + path: osrd-editoast-test.tar + - name: Upload editoast artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: editoast + path: osrd-editoast.tar + - name: Upload gateway-test artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: gateway-test + path: osrd-gateway-test.tar + - name: Upload front-tests artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: front-tests + path: osrd-front-tests.tar + - name: Upload front artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: front + path: osrd-front.tar + - name: Upload core artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: core + path: osrd-core.tar + - name: Upload gateway-standalone artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: gateway-standalone + path: osrd-gateway-standalone.tar + - name: Upload front-nginx artifact + uses: actions/upload-artifact@v4 + if: steps.bake-metadata.outputs.output_method == 'artifact' + with: + name: front-nginx + path: osrd-front-nginx.tar + check_generated_railjson_sync: runs-on: ubuntu-latest steps: @@ -332,6 +394,19 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built images + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: front-build + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-front-build.tar + docker image ls -a + - name: Generate rtk bindings run: > docker run --rm --net=host -v $PWD/output:/app/tests/unit @@ -353,6 +428,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built images + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: core-build + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-core-build.tar + - name: Execute tests within container run: | docker run --name core-test \ @@ -418,6 +505,25 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built front-build image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: front-build + path: . + - name: Download built editoast-test image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: editoast-test + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-front-build.tar + docker load --input ./osrd-editoast-test.tar + - name: Check for i18n missing keys run: | docker run --name=front-i18n-checker --net=host \ @@ -461,6 +567,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built images + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: editoast-test + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-editoast-test.tar + - name: Documentation check run: | docker run --name=editoast-doc --net=host -v $PWD/output:/output \ @@ -496,6 +614,25 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built editoast image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: editoast + path: . + - name: Download built front-build image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: front-build + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-editoast.tar + docker load --input ./osrd-front-build.tar + - name: Generate OpenAPI run: | docker run --name=editoast-test --net=host -v $PWD/output:/output \ @@ -522,6 +659,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built images + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: gateway-test + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-gateway-test.tar + - name: Execute tests within container run: | docker run --name=gateway-test --net=host -v $PWD/output:/output \ @@ -572,6 +721,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download built images + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: front-build + path: . + + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-front-build.tar + - name: Execute tests within container run: | docker run --name=front-test --net=host -v $PWD/output:/app/tests/unit \ @@ -599,6 +760,44 @@ jobs: # https://www.jameskerr.blog/posts/sharing-steps-in-github-action-workflows/ - name: Checkout uses: actions/checkout@v4 + - name: Download built front-tests image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: front-tests + path: . + - name: Download built editoast image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: editoast + path: . + - name: Download built core image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: core + path: . + - name: Download built gateway-standalone image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: gateway-standalone + path: . + - name: Download built front-nginx image + if: needs.build.outputs.output_method == 'artifact' + uses: actions/download-artifact@v4 + with: + name: front-nginx + path: . + - name: Load built images + if: needs.build.outputs.output_method == 'artifact' + run: | + docker load --input ./osrd-front-tests.tar + docker load --input ./osrd-editoast.tar + docker load --input ./osrd-core.tar + docker load --input ./osrd-gateway-standalone.tar + docker load --input ./osrd-front-nginx.tar - name: Install poetry run: pipx install poetry - name: Set up Python @@ -618,7 +817,7 @@ jobs: export OSRD_FRONT_MODE=nginx export TAG='${{ needs.build.outputs.stable_version }}' services='editoast core front gateway' - docker compose pull $services + docker compose pull --policy missing $services docker compose up --no-build -d $services jaeger env: DOCKER_BUILDKIT: 1