TiDB readiness probes (pingcap#147)

* add readiness probes for tidb service
xiaojingchen · Oct 31, 2018 · ab2f1c2 · ab2f1c2
1 parent fd73d27
commit ab2f1c2
Show file tree

Hide file tree

Showing 5 changed files with 150 additions and 0 deletions.
diff --git a/charts/tidb-cluster/templates/restore-job.yaml b/charts/tidb-cluster/templates/restore-job.yaml
@@ -0,0 +1,78 @@
+{{- if .Values.restore.create }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Values.clusterName }}-restore
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: restore
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "chart.name" . }}
+        app.kubernetes.io/instance: {{ .Values.clusterName }}
+        app.kubernetes.io/component: restore
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: tidb-restore-job
+        image: {{ .Values.restore.image }}
+        command:
+        - /bin/sh
+        - -c
+        - |
+          set -euo pipefail
+          host=`echo {{ .Values.clusterName }}_TIDB_SERVICE_HOST | tr '[a-z]' '[A-Z]'`
+        {{- if .Values.restore.gcp }}
+          downloader \
+            --cloud=gcp \
+            --bucket={{ .Values.restore.gcp.bucket }} \
+            --srcDir={{ .Values.restore.gcp.srcDir }} \
+            --destDir=/data
+
+          dataDir=/data/{{ .Values.restore.gcp.srcDir }}
+          ./loader \
+            -d ${dataDir} \
+            -h `eval echo '${'$host'}'` \
+            -u {{ .Values.restore.user }} \
+            -p ${TIDB_PASSWORD} \
+            -P 4000 \
+            {{ .Values.restore.options }}
+        {{- end }}
+        volumeMounts:
+        - name: data
+          mountPath: "/data"
+      {{- if .Values.restore.gcp }}
+        - name: gcp-credentials
+          mountPath: "/gcp"
+          readOnly: true
+      {{- end }}
+        env:
+        - name: MY_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+      {{- if .Values.restore.gcp }}
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /gcp/credentials.json
+      {{- end }}
+        - name: TIDB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.clusterName }}-restore-secret
+              key: user-password
+        restartPolicy: OnFailure
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: {{ .Values.clusterName }}-restore
+    {{- if .Values.restore.gcp }}
+      - name: gcp-credentials
+        secret:
+          secretName: {{ .Values.clusterName }}-restore-secret
+    {{- end }}
+{{- end }}
diff --git a/charts/tidb-cluster/templates/restore-pvc.yaml b/charts/tidb-cluster/templates/restore-pvc.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.restore.create }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.clusterName }}-restore
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: tidb-operator
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: restore
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: {{ .Values.restore.storage }}
+  storageClassName: {{ .Values.restore.storageClassName }}
+{{- end }}
diff --git a/charts/tidb-cluster/templates/restore-secret.yaml b/charts/tidb-cluster/templates/restore-secret.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.restore.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.clusterName }}-restore-secret
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Values.clusterName }}
+    app.kubernetes.io/component: restore-secret
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+type: Opaque
+data:
+{{- if .Values.restore.gcp }}
+  credentials.json: {{ .Values.restore.gcp.credentialsData }}
+{{- end }}
+  user-password: {{ .Values.restore.password | b64enc | quote }}
+{{- end }}
diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml
@@ -279,6 +279,31 @@ binlog:
       # kafkaAddrs: "127.0.0.1:9092"
       # kafkaVersion: "0.8.2.0"
 
+restore:
+  create: false
+  # /~https://github.com/tennix/tidb-cloud-backup
+  image: pingcap/tidb-cloud-backup:latest
+  # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer.
+  # different classes might map to quality-of-service levels, or to backup policies,
+  # or to arbitrary policies determined by the cluster administrators.
+  # refer to https://kubernetes.io/docs/concepts/storage/storage-classes
+  storageClassName: local-storage
+  storage: 1000Gi
+  # loader options https://www.pingcap.com/docs-cn/tools/loader/
+  options: "-t 16"
+  # user must set tidb cluster user and password
+  user: restore
+  password: "Password here ..."
+  # download backup data from gcp
+  gcp:
+    # credentialsData is base64 format of the service account json file data
+    # https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually
+    # and then: cat credentials.json | base64 | tr -d '\n'
+    # credentialsData: ""
+    # bucket: ""
+    # srcDir is the direction of backup data in bucket
+    # srcDir: ""
+
 metaInstance: "{{ $labels.instance }}"
 metaType: "{{ $labels.type }}"
 metaValue: "{{ $value }}"
diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go
@@ -317,6 +317,15 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust
 									Value: strconv.FormatBool(tc.Spec.TiDB.BinlogEnabled),
 								},
 							},
+							ReadinessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/status",
+										Port: intstr.FromInt(10080),
+									},
+								},
+								InitialDelaySeconds: int32(10),
+							},
 						},
 					},
 					RestartPolicy: corev1.RestartPolicyAlways,