Merge branch 'master' into fix_recorded_rule_deduplication

armadaproject · Feb 27, 2025 · a373ce5 · a373ce5
2 parents 0e0fca4 + 7e1e21e
commit a373ce5
Show file tree

Hide file tree

Showing 235 changed files with 4,174 additions and 1,096 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -23,9 +23,9 @@ jobs:
           cache: yarn
           cache-dependency-path: ./internal/lookoutui/yarn.lock
 
-      - name: Install Dependencies And Run Unit Tests
+      - name: Install Dependencies, Check Types And Run Unit Tests
         run: |
-          yarn install --frozen-lockfile && yarn openapi && CI=true yarn test --reporter=junit
+          yarn install --frozen-lockfile && yarn openapi && yarn typecheck && CI=true yarn test --reporter=junit
         working-directory: ./internal/lookoutui
 
       - name: Publish JUnit Report

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -68,18 +68,18 @@ builds:
     goarch:
       - amd64
   - env: [CGO_ENABLED=0]
-    id: lookoutv2
-    binary: lookoutv2
-    main: ./cmd/lookoutv2/main.go
+    id: lookout
+    binary: lookout
+    main: ./cmd/lookout/main.go
     mod_timestamp: '{{ .CommitTimestamp }}'
     goos:
       - linux
     goarch:
       - amd64
   - env: [CGO_ENABLED=0]
-    id: lookoutingesterv2
-    binary: lookoutingesterv2
-    main: ./cmd/lookoutingesterv2/main.go
+    id: lookoutingester
+    binary: lookoutingester
+    main: ./cmd/lookoutingester/main.go
     mod_timestamp: '{{ .CommitTimestamp }}'
     goos:
       - linux
@@ -215,12 +215,12 @@ dockers:
       - "{{ .Env.DOCKER_REPO }}armada-lookout-bundle:{{ .Version }}"
     build_flag_templates: *BUILD_FLAG_TEMPLATES
     ids:
-      - lookoutv2
-      - lookoutingesterv2
+      - lookout
+      - lookoutingester
     extra_files:
       - config/logging.yaml
-      - config/lookoutv2/config.yaml
-      - config/lookoutingesterv2/config.yaml
+      - config/lookout/config.yaml
+      - config/lookoutingester/config.yaml
       - internal/lookoutui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
@@ -243,8 +243,8 @@ dockers:
       - armadactl
       - scheduler
       - scheduleringester
-      - lookoutv2
-      - lookoutingesterv2
+      - lookout
+      - lookoutingester
       - fakeexecutor
     extra_files:
       - config/logging.yaml
@@ -254,8 +254,8 @@ dockers:
       - config/eventingester/config.yaml
       - config/scheduler/config.yaml
       - config/scheduleringester/config.yaml
-      - config/lookoutv2/config.yaml
-      - config/lookoutingesterv2/config.yaml
+      - config/lookout/config.yaml
+      - config/lookoutingester/config.yaml
       - internal/lookoutui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
@@ -331,41 +331,45 @@ dockers:
       - testsuite
     dockerfile: ./build/testsuite/Dockerfile
 
-  - id: lookoutingesterv2
+  - id: lookoutingester
     use: buildx
     goos: linux
     goarch: amd64
     image_templates:
       - "{{ .Env.DOCKER_REPO }}armada-lookout-ingester-v2:latest"
       - "{{ .Env.DOCKER_REPO }}armada-lookout-ingester-v2:{{ .Version }}"
+      - "{{ .Env.DOCKER_REPO }}armada-lookout-ingester:latest"
+      - "{{ .Env.DOCKER_REPO }}armada-lookout-ingester:{{ .Version }}"
     build_flag_templates: *BUILD_FLAG_TEMPLATES
     ids:
-      - lookoutingesterv2
+      - lookoutingester
     extra_files:
-      - config/lookoutingesterv2/config.yaml
+      - config/lookoutingester/config.yaml
       - config/logging.yaml
-    dockerfile: ./build/lookoutingesterv2/Dockerfile
+    dockerfile: ./build/lookoutingester/Dockerfile
 
-  - id: lookoutv2
+  - id: lookout
     use: buildx
     goos: linux
     goarch: amd64
     image_templates:
       - "{{ .Env.DOCKER_REPO }}armada-lookout-v2:latest"
       - "{{ .Env.DOCKER_REPO }}armada-lookout-v2:{{ .Version }}"
+      - "{{ .Env.DOCKER_REPO }}armada-lookout:latest"
+      - "{{ .Env.DOCKER_REPO }}armada-lookout:{{ .Version }}"
     build_flag_templates: *BUILD_FLAG_TEMPLATES
     ids:
-      - lookoutv2
-      - lookoutingesterv2
+      - lookout
+      - lookoutingester
     extra_files:
       - internal/lookoutui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
       - pkg/api/schedulerobjects/api.swagger.json
-      - config/lookoutv2/config.yaml
-      - config/lookoutingesterv2/config.yaml
+      - config/lookout/config.yaml
+      - config/lookoutingester/config.yaml
       - config/logging.yaml
-    dockerfile: ./build/lookoutv2/Dockerfile
+    dockerfile: ./build/lookout/Dockerfile
 
   - id: eventingester
     use: buildx
@@ -500,12 +504,16 @@ release:
     #### Armada Executor
     - `docker pull {{ .Env.DOCKER_REPO }}armada-executor:{{ .Version }}`
     - `docker pull {{ .Env.DOCKER_REPO }}armada-executor:latest`
-    #### Armada Lookout V2
+    #### Armada Lookout
     - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout-v2:{{ .Version }}`
     - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout-v2:latest`
-    #### Armada Lookout Ingester V2
+    - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout:{{ .Version }}`
+    - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout:latest`
+    #### Armada Lookout Ingester
     - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout-ingester-v2:{{ .Version }}`
     - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout-ingester-v2:latest`
+    - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout-ingester:{{ .Version }}`
+    - `docker pull {{ .Env.DOCKER_REPO }}armada-lookout-ingester:latest`
     #### Armada Event Ingester
     - `docker pull {{ .Env.DOCKER_REPO }}armada-event-ingester:{{ .Version }}`
     - `docker pull {{ .Env.DOCKER_REPO }}armada-event-ingester:latest`

diff --git a/.mergify.yml b/.mergify.yml
@@ -9,6 +9,6 @@ pull_request_rules:
           - "#approved-reviews-by>=2"
           - and: 
             - "#approved-reviews-by>=1"
-            - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|geaere|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx)"
+            - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|geaere|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx|EnricoMi)$"
         title:
-          Two are checks required.
+          Two approvals required, or one if author is a maintainer.
diff --git a/.run/Armada Infrastructure Services.run.xml b/.run/Armada Infrastructure Services.run.xml
@@ -5,15 +5,13 @@
         <option name="envFilePath" value="" />
         <option name="services">
           <list>
-            <option value="lookoutv2-migration" />
+            <option value="lookout-migration" />
             <option value="scheduler-migration" />
           </list>
         </option>
         <option name="sourceFilePath" value="docker-compose.yaml" />
       </settings>
     </deployment>
-    <method v="2">
-      <option name="RunConfigurationTask" enabled="true" run_configuration_name="Armada Infrastructure Services" run_configuration_type="docker-deploy" />
-    </method>
+    <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Armada.run.xml b/.run/Armada.run.xml
@@ -2,11 +2,11 @@
   <configuration default="false" name="Armada" type="CompoundRunConfigurationType">
     <toRun name="Event Ingester" type="GoApplicationRunConfiguration" />
     <toRun name="Executor" type="GoApplicationRunConfiguration" />
-    <toRun name="Lookout Ingester V2" type="GoApplicationRunConfiguration" />
-    <toRun name="LookoutV2" type="GoApplicationRunConfiguration" />
+    <toRun name="Lookout Ingester" type="GoApplicationRunConfiguration" />
+    <toRun name="Lookout" type="GoApplicationRunConfiguration" />
     <toRun name="Scheduler" type="GoApplicationRunConfiguration" />
     <toRun name="Scheduler Ingester" type="GoApplicationRunConfiguration" />
     <toRun name="Server" type="GoApplicationRunConfiguration" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Lookout Ingester V2.run.xml → .run/Lookout Ingester.run.xml b/.run/Lookout Ingester V2.run.xml → .run/Lookout Ingester.run.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Lookout Ingester V2" type="GoApplicationRunConfiguration" factoryName="Go Application">
+  <configuration default="false" name="Lookout Ingester" type="GoApplicationRunConfiguration" factoryName="Go Application">
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>
@@ -10,9 +10,9 @@
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
     </envs>
     <kind value="FILE" />
-    <package value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
+    <package value="$PROJECT_DIR$/cmd/lookout/main.go" />
     <directory value="$PROJECT_DIR$" />
-    <filePath value="$PROJECT_DIR$/cmd/lookoutingesterv2/main.go" />
+    <filePath value="$PROJECT_DIR$/cmd/lookoutingester/main.go" />
     <method v="2" />
   </configuration>
 </component>
diff --git a/.run/LookoutV2 UI.run.xml → .run/Lookout UI.run.xml b/.run/LookoutV2 UI.run.xml → .run/Lookout UI.run.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="LookoutV2 UI" type="ShConfigurationType">
+  <configuration default="false" name="Lookout UI" type="ShConfigurationType">
     <option name="SCRIPT_TEXT" value="" />
     <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
     <option name="SCRIPT_PATH" value="$PROJECT_DIR$/internal/lookoutui/development-setup-and-start.sh" />

diff --git a/.run/LookoutV2.run.xml → .run/Lookout.run.xml b/.run/LookoutV2.run.xml → .run/Lookout.run.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="LookoutV2" type="GoApplicationRunConfiguration" factoryName="Go Application">
+  <configuration default="false" name="Lookout" type="GoApplicationRunConfiguration" factoryName="Go Application">
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>
@@ -11,9 +11,9 @@
       <env name="ARMADA_UICONFIG_BINOCULARSBASEURLPATTERN" value="http://localhost:8082" />
     </envs>
     <kind value="FILE" />
-    <package value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
+    <package value="$PROJECT_DIR$/cmd/lookout/main.go" />
     <directory value="$PROJECT_DIR$" />
-    <filePath value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
+    <filePath value="$PROJECT_DIR$/cmd/lookout/main.go" />
     <method v="2" />
   </configuration>
 </component>
diff --git a/.run/lookoutv2PostgresMigration.run.xml → .run/lookoutPostgresMigration.run.xml b/.run/lookoutv2PostgresMigration.run.xml → .run/lookoutPostgresMigration.run.xml
@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="lookoutv2PostgresMigration" type="GoApplicationRunConfiguration" factoryName="Go Application">
+  <configuration default="false" name="lookoutPostgresMigration" type="GoApplicationRunConfiguration" factoryName="Go Application">
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <parameters value="--migrateDatabase" />
@@ -10,9 +10,9 @@
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
     </envs>
     <kind value="FILE" />
-    <package value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
+    <package value="$PROJECT_DIR$/cmd/lookout/main.go" />
     <directory value="$PROJECT_DIR$" />
-    <filePath value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />
+    <filePath value="$PROJECT_DIR$/cmd/lookout/main.go" />
     <method v="2" />
   </configuration>
 </component>
diff --git a/build/bundles/full/Dockerfile b/build/bundles/full/Dockerfile
@@ -23,7 +23,7 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
 COPY --from=NODE /lookoutui/build/ /app/internal/lookoutui/build
 
-COPY server executor binoculars eventingester lookoutingesterv2 lookoutv2 scheduler scheduleringester armadactl /app/
+COPY server executor binoculars eventingester lookoutingester lookout scheduler scheduleringester armadactl /app/
 COPY config/ /app/config/
 
 WORKDIR /app
diff --git a/build/bundles/lookout/Dockerfile b/build/bundles/lookout/Dockerfile
@@ -11,10 +11,10 @@ USER armada
 # Logging (shared between both components)
 COPY config/logging.yaml /app/config/logging.yaml
 
-COPY config/lookoutingesterv2/config.yaml /app/config/lookoutingesterv2/config.yaml
+COPY config/lookoutingester/config.yaml /app/config/lookoutingester/config.yaml
 
-COPY config/lookoutv2/config.yaml /app/config/lookoutv2/config.yaml
+COPY config/lookout/config.yaml /app/config/lookout/config.yaml
 
-COPY lookoutingesterv2 lookoutv2 /app/
+COPY lookoutingester lookout /app/
 
 WORKDIR /app
diff --git a/build/lookoutv2/Dockerfile → build/lookout/Dockerfile b/build/lookoutv2/Dockerfile → build/lookout/Dockerfile
@@ -18,15 +18,15 @@ RUN yarn build
 
 FROM ${BASE_IMAGE}
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
-LABEL org.opencontainers.image.title=lookoutv2
-LABEL org.opencontainers.image.description="Lookout V2"
-LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/lookoutv2
+LABEL org.opencontainers.image.title=lookout
+LABEL org.opencontainers.image.description="Lookout"
+LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/lookout
 USER armada
 COPY --from=NODE /lookoutui/build/ /app/internal/lookoutui/build
-COPY lookoutv2 /app/
-COPY config/lookoutv2/config.yaml /app/config/lookoutv2/config.yaml
-COPY lookoutingesterv2 /app/
-COPY config/lookoutingesterv2/ /app/config/lookoutingesterv2
+COPY lookout /app/
+COPY config/lookout/config.yaml /app/config/lookout/config.yaml
+COPY lookoutingester /app/
+COPY config/lookoutingester/ /app/config/lookoutingester
 COPY config/logging.yaml /app/config/logging.yaml
 WORKDIR /app
-ENTRYPOINT ["./lookoutv2"]
+ENTRYPOINT ["./lookout"]
diff --git a/build/lookoutingester/Dockerfile b/build/lookoutingester/Dockerfile
@@ -0,0 +1,17 @@
+ARG BASE_IMAGE=alpine:3.21.0
+
+FROM ${BASE_IMAGE}
+LABEL org.opencontainers.image.title=lookoutingester
+LABEL org.opencontainers.image.description="Lookout Ingester"
+LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/lookoutingester
+
+RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
+USER armada
+
+COPY lookoutingester /app/
+COPY config/lookoutingester/config.yaml /app/config/lookoutingester/config.yaml
+COPY config/logging.yaml /app/config/logging.yaml
+
+WORKDIR /app
+
+ENTRYPOINT ["./lookoutingester"]
diff --git a/build/lookoutingesterv2/Dockerfile b/build/lookoutingesterv2/Dockerfile
diff --git a/cmd/lookout/dbloadtester/config.yaml b/cmd/lookout/dbloadtester/config.yaml
@@ -0,0 +1,10 @@
+#sample config
+---
+queues:
+  - "queue1"
+ids:
+  - "01JD2FF8TTJAXCNFB0F91053ZD"
+jobsets:
+  - "jobset1"
+queueCounts:
+  queue1: 1000000