pod belonging bug (#27)

* docs: update status machine

* fix: specify pod by task name instead of task type

* chore: update version to v1.1.3

.github/workflows/release.yaml

@@ -6,7 +6,7 @@ name: Release
 on: [push]
 
 env:
-  version: v1.1.2
+  version: v1.1.3
 
 jobs:
   docker:

Makefile

@@ -1,7 +1,7 @@
 
 # di-operator version
 APP_VERSION ?= 0.1.0
-VERSION ?= v1.1.2
+VERSION ?= v1.1.3
 MASTER_VERSION := $(VERSION)
 
 COMMIT_SHORT_SHA=$(shell git log -n 1 | head -n 1 | sed -e 's/^commit //' | head -c 8)

chart/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 1.1.2
+version: 1.1.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

chart/values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 
 # tag for pytorch-operator image
-tag: v1.1.2
+tag: v1.1.3
 
 # tag for di-orchestrator image
 registry: opendilab

config/di-manager.yaml

@@ -5214,7 +5214,7 @@ spec:
         envFrom:
         - configMapRef:
             name: di-config
-        image: opendilab/di-orchestrator:v1.1.2
+        image: opendilab/di-orchestrator:v1.1.3
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -5270,7 +5270,7 @@ spec:
         envFrom:
         - configMapRef:
             name: di-config
-        image: opendilab/di-orchestrator:v1.1.2
+        image: opendilab/di-orchestrator:v1.1.3
         imagePullPolicy: Always
         livenessProbe:
           httpGet:

config/manager/di_operator.yaml

@@ -26,7 +26,7 @@ spec:
         - "--leader-elect"
         - "--qps=100"
         - "--burst=200"
-        image: opendilab/di-orchestrator:v1.1.2
+        image: opendilab/di-orchestrator:v1.1.3
         imagePullPolicy: Always
         name: manager
         envFrom:

config/manager/di_server.yaml

@@ -24,7 +24,7 @@ spec:
         - "--server-bind-address=:8081"
         - "--qps=100"
         - "--burst=200"
-        image: opendilab/di-orchestrator:v1.1.2
+        image: opendilab/di-orchestrator:v1.1.3
         imagePullPolicy: Always
         name: server
         envFrom:

config/manager/di_webhook.yaml

@@ -24,7 +24,7 @@ spec:
         - "--port=9443"
         - "--qps=100"
         - "--burst=200"
-        image: opendilab/di-orchestrator:v1.1.2
+        image: opendilab/di-orchestrator:v1.1.3
         imagePullPolicy: Always
         name: webhook
         securityContext:

config/manager/kustomization.yaml

@@ -8,4 +8,4 @@ kind: Kustomization
 images:
 - name: opendilab/di-orchestrator
   newName: opendilab/di-orchestrator
-  newTag: v1.1.2
+  newTag: v1.1.3

pkg/controllers/dijob.go

@@ -279,7 +279,7 @@ func (r *DIJobReconciler) createMissedReplicas(ctx context.Context, job *div2alp
 	for index, task := range job.Spec.Tasks {
 		localTrace := make([]bool, task.Replicas)
 		for _, pod := range pods {
-			if pod.Labels[dicommon.LabelTaskType] != string(task.Type) {
+			if pod.Labels[dicommon.LabelTaskName] != string(task.Name) {
 				continue
 			}
 			localRank, err := strconv.Atoi(pod.Annotations[dicommon.AnnotationTaskRank])
@@ -288,6 +288,11 @@ func (r *DIJobReconciler) createMissedReplicas(ctx context.Context, job *div2alp
 					pod.Annotations[dicommon.AnnotationTaskRank]))
 				return err
 			}
+			if localRank >= len(localTrace) {
+				err := fmt.Errorf("pod %s local rank is out of range", pod.Name)
+				log.Error(err, "get local rank")
+				return err
+			}
 			localTrace[localRank] = true
 		}
 		for localRank := 0; localRank < int(task.Replicas); localRank++ {