pipeline health POC

.github/workflows/repos.yml

@@ -0,0 +1,64 @@
+name: Preview or update Pulumi app nf-core/github-repos/prod
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "pulumi/github/repos/**/*"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "pulumi/github/repos/**/*"
+
+# These are the environment variables that jobs in the workflow have access to.
+# By defining them here, all jobs/steps will have access to these variables.
+env:
+  # IMPORTANT! You must map the env vars for your cloud provider here even though you add them as secrets
+  # to this repository.
+  # See the setup page for cloud providers here: https://www.pulumi.com/docs/intro/cloud-providers/.
+  # For example, if you are using AWS, then you should add the following:
+  # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+  PULUMI_STACK_NAME: nf-core/github-repos/prod
+  PULUMI_WORKING_DIRECTORY: pulumi/github/repos/
+
+jobs:
+  pulumi:
+    name: Pulumi
+    runs-on: ubuntu-latest
+    steps:
+      # Turnstyle is used to prevent multiple push jobs from running at the same time. We
+      # limit it to push jobs to allow PR jobs to run concurrently.
+      - name: Turnstyle
+        if: ${{ github.event_name == 'push' }}
+        uses: softprops/turnstyle@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/checkout@v2
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        cache: 'pip'
+
+      - run: pip install -r requirements.txt
+        working-directory: ${{ env.PULUMI_WORKING_DIRECTORY }}
+
+      - name: PR previews
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: pulumi/actions@v3
+        with:
+          command: preview
+          stack-name: ${{ env.PULUMI_STACK_NAME }}
+          work-dir: ${{ env.PULUMI_WORKING_DIRECTORY }}
+
+      - name: Apply infrastructure update
+        if: ${{ github.event_name == 'push' }}
+        uses: pulumi/actions@v3
+        with:
+          command: up
+          stack-name: ${{ env.PULUMI_STACK_NAME }}
+          work-dir: ${{ env.PULUMI_WORKING_DIRECTORY }}
+

docs/1password.md

@@ -0,0 +1,28 @@
+# Pulumi
+
+[Pulumi Shell Plugin](https://developer.1password.com/docs/cli/shell-plugins/pulumi/)
+
+[How to use 1Password with different accounts automatically](https://developer.1password.com/docs/cli/shell-plugins/multiple-accounts/)
+
+```console
+cd ~/src/nf-core
+
+op signin
+
+# Select nf-core
+
+op plugin init pulumi
+```
+
+This should result in:
+```
+
+Pulumi CLI
+Authenticate with Pulumi Personal Access Token.
+
+? Locate your Pulumi Personal Access Token: Search in 1Password...
+
+? Locate your Pulumi Personal Access Token: Pulumi Personal Access Token (Private)
+
+? Configure when the chosen credential(s) will be used to authenticate: Use automatically when in this directory or subdirectories
+```

docs/pulumi.md

@@ -0,0 +1,72 @@
+# Pulumi
+
+nf-core specific docs, links and guides.
+
+## Quick Start
+
+### Repo structure
+
+This repo is a "Monorepo", basically a bunch of smaller projects inside of one bigger project.
+
+```console
+tree -L 1 pulumi
+pulumi
+├── AWSMegatests
+├── github
+├── repo-backups
+├── sentieon-license-server
+└── test-datasets
+```
+
+Each of these are their own projects.
+
+### Install Pulumi
+
+[Here's the official guide](https://www.pulumi.com/docs/clouds/aws/get-started/)
+
+### Working with this repo
+
+<!-- TODO Need someone fresh to run through this so we can see where the pain points are-->
+
+1. Open up the project you want to make a change to
+2. Make the change (Probably in `__main.py__`)
+3. If you have a Pulumi cloud account in the nf-core org `pulumi preview` and `pulumi up` should work locally
+4. Create a branch in the repo and make a PR, and a preview of the deployment should get ran.
+
+## Terminology
+
+Pulumi is pretty heavy on the terms and it was kinda confusing. A hierarchy kinda helps
+
+```
+Projects
+├── Stacks
+├──── Deployments
+├──── Resources
+Environments
+```
+
+### Projects
+
+Each directory in `pulumi` is a project.
+
+#### Stacks
+
+Each project can have multiple stacks. For example, `dev`, `prod`, `test`.
+
+Official quote:
+
+> What are projects and stacks? Pulumi projects and stacks let you organize Pulumi code. Consider a Pulumi project to be analogous to a GitHub repo—a single place for code—and a stack to be an instance of that code with a separate configuration. For instance, project foo may have multiple stacks for different deployment environments (dev, test, or prod), or perhaps for different cloud configurations (geographic region for example). See Organizing Projects and Stacks for some best practices on organizing your Pulumi projects and stacks.
+
+https://www.pulumi.com/docs/using-pulumi/organizing-projects-stacks/
+
+##### Deployments
+
+Everytime you push to main in this repo a new deployment of the stack goes out.
+
+##### Resources
+
+These are individual pieces of infrastructure. An EC2 instance, a VPC, a GitHub repo, a GitHub team are some examples.
+
+### Environments
+
+This is Pulumi's hosted Secrete management. I'm thinking of these like, well "Environments". The nf-core AWS, the nf-core Azure, nf-core GCP, nf-core GitHub org, the nf-core-tf GitHub org.

pulumi/github/repos/.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+venv/
+__pycache__/

pulumi/github/repos/Pulumi.dev.yaml

@@ -0,0 +1,5 @@
+config:
+  github:owner: nf-core-tf
+  # https://start.1password.com/open/i?a=O5GICFDKPNABLLVGMKBL5JWDWA&v=rdfcz6oy6qxxrc4clu467a7dmm&i=4ajrv44kc5lcbboa37fr5oydla&h=nf-core.1password.eu
+environment:
+  - github-nf-core-tf

pulumi/github/repos/Pulumi.prod.yaml

@@ -0,0 +1,5 @@
+config:
+  github:owner: nf-core
+# https://start.1password.com/open/i?a=O5GICFDKPNABLLVGMKBL5JWDWA&v=rdfcz6oy6qxxrc4clu467a7dmm&i=ttqz63qvlr5qfwfde424nbl4re&h=nf-core.1password.eu
+environment:
+  - github-prod

pulumi/github/repos/Pulumi.yaml

@@ -0,0 +1,6 @@
+name: github-repos
+runtime:
+  name: python
+  options:
+    virtualenv: venv
+description: Managing nf-core GitHub repos

pulumi/github/repos/README.md

@@ -0,0 +1,18 @@
+# Repos
+
+Replaces the automatic rule enforcement from the [old Pipeline Health PHP code](/~https://github.com/nf-core/website/blob/old-site/public_html/pipeline_health.php),
+
+[Main GitHub Issue](/~https://github.com/nf-core/ops/issues/5)
+[Tracking Milestone](/~https://github.com/nf-core/ops/milestone/1)
+
+## Useful Docs
+
+- https://www.pulumi.com/registry/packages/github/api-docs/repository/
+- [Old Pipeline Health PHP code](/~https://github.com/nf-core/website/blob/old-site/public_html/pipeline_health.php)
+- [New Pipeline Health page](/~https://github.com/nf-core/website/blob/main/sites/pipelines/src/pages/pipeline_health.astro)
+
+### Importing Repos
+
+```sh
+pulumi env run nf-core/github-prod -i pulumi import github:index/repository:Repository testpipeline testpipeline
+```

pulumi/github/repos/__main__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+import pipelines.testpipeline

pulumi/github/repos/core/modules.py

@@ -0,0 +1,42 @@
+import yaml
+
+import pulumi
+import pulumi_github as github
+
+
+nf_core_tf = github.Repository(
+    "nf-core-tf",
+    allow_merge_commit=False,
+    allow_rebase_merge=False,
+    allow_squash_merge=False,
+    default_branch="master",
+    description="Repository to host tool-specific module files for the Nextflow DSL2 community!",
+    has_downloads=True,
+    has_issues=True,
+    has_projects=True,
+    homepage_url="https://nf-co.re",
+    merge_commit_message="",
+    merge_commit_title="",
+    name="modules",
+    security_and_analysis=github.RepositorySecurityAndAnalysisArgs(
+        secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs(
+            status="disabled",
+        ),
+        secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs(
+            status="disabled",
+        ),
+    ),
+    squash_merge_commit_message="",
+    squash_merge_commit_title="",
+    topics=[
+        "nextflow",
+        "pipelines",
+        "nf-test",
+        "modules",
+        "nf-core",
+        "dsl2",
+        "workflows",
+    ],
+    visibility="public",
+    opts=pulumi.ResourceOptions(protect=True),
+)

pulumi/github/repos/core_repos.yml

@@ -0,0 +1,16 @@
+- .github
+- basic_training
+- configs
+- logos
+- modules
+- ops
+- prettier-plugin-nextflow
+- references
+- setup-nextflow
+- setup-nf-test
+- sublime
+- test-datasets
+- tools
+- vale
+- vscode-extensionpack
+- website

pulumi/github/repos/import_by_hand.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import pulumi
+import pulumi_github as github
+
+import pipelines.denovotranscript
+import pipelines.meerpipe
+import pipelines.pairgenomealign
+import pipelines.phaseimpute
+import pipelines.reportho
+
+# ...
+
+import core.github
+import core.modules
+
+# ...
+import core.website

pulumi/github/repos/loop_example.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import pulumi
+import pulumi_github as github
+
+TOPICS = [
+    "nextflow",
+    "pipelines",
+    "nf-test",
+    "modules",
+    "nf-core",
+    "dsl2",
+    "workflows",
+]
+
+alpha_test_pipeline_repos = [
+    "denovotranscript",
+    "meerpipe",
+    "pairgenomealign",
+    "phaseimpute",
+    "reportho",
+]
+
+for pipeline in alpha_test_pipeline_repos:
+    github.Repository(
+        pipeline,
+        allow_merge_commit=True,
+        allow_rebase_merge=True,
+        allow_squash_merge=True,
+        default_branch="master",
+        description="Alpha test repository for nf-core",
+        has_downloads=True,
+        has_issues=True,
+        has_projects=True,
+        homepage_url=f"https://nf-co.re/{pipeline}",
+        merge_commit_message="",
+        merge_commit_title="",
+        name=pipeline,
+        security_and_analysis=github.RepositorySecurityAndAnalysisArgs(
+            secret_scanning=github.RepositorySecurityAndAnalysisSecretScanningArgs(
+                status="disabled",
+            ),
+            secret_scanning_push_protection=github.RepositorySecurityAndAnalysisSecretScanningPushProtectionArgs(
+                status="disabled",
+            ),
+        ),
+        squash_merge_commit_message="",
+        squash_merge_commit_title="",
+        topics=TOPICS,
+        visibility="public",
+        # NOTE Idk if this will work
+        opts=pulumi.ResourceOptions(protect=True),
+    )