diff --git a/deployment/aws_ecs_fargate/cloudformation/uninstall.sh b/deployment/aws_ecs_fargate/cloudformation/uninstall.sh index 898fb1212ae..0965f6d045b 100755 --- a/deployment/aws_ecs_fargate/cloudformation/uninstall.sh +++ b/deployment/aws_ecs_fargate/cloudformation/uninstall.sh @@ -23,7 +23,7 @@ delete_stack() { if [ "$stack_name" == "${ENVIRONMENT}-onyx-cluster" ]; then echo "Removing all objects and directories from the onyx config s3 bucket." - aws s3 rm "s3://${ENVIRONMENT}-${S3_BUCKET} --recursive" + aws s3 rm "s3://${ENVIRONMENT}-${S3_BUCKET}" --recursive sleep 5 fi diff --git a/deployment/cloud_kubernetes/hpa/workers_hpa.yaml b/deployment/cloud_kubernetes/hpa/workers_hpa.yaml index fd24b9eeac3..5e35da9f088 100644 --- a/deployment/cloud_kubernetes/hpa/workers_hpa.yaml +++ b/deployment/cloud_kubernetes/hpa/workers_hpa.yaml @@ -54,3 +54,22 @@ spec: target: type: Utilization averageUtilization: 70 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: celery-worker-monitoring-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: celery-worker-indexing + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 diff --git a/deployment/cloud_kubernetes/workers/monitoring.yaml b/deployment/cloud_kubernetes/workers/monitoring.yaml new file mode 100644 index 00000000000..6ccb7c99745 --- /dev/null +++ b/deployment/cloud_kubernetes/workers/monitoring.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-worker-monitoring +spec: + replicas: 2 + selector: + matchLabels: + app: celery-worker-monitoring + template: + metadata: + labels: + app: celery-worker-monitoring + spec: + containers: + - name: celery-worker-monitoring + image: onyxdotapp/onyx-backend-cloud:v0.14.0-cloud.beta.21 + imagePullPolicy: IfNotPresent + command: + [ + "celery", + "-A", + "onyx.background.celery.versioned_apps.monitoring", + "worker", + "--loglevel=INFO", + "--hostname=monitoring@%n", + "-Q", + "monitoring", + "--prefetch-multiplier=8", + "--concurrency=8", + ] + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: onyx-secrets + key: redis_password + - name: ONYX_VERSION + value: "v0.11.0-cloud.beta.8" + envFrom: + - configMapRef: + name: env-configmap + volumeMounts: + - name: vespa-certificates + mountPath: "/app/certs" + readOnly: true + resources: + requests: + cpu: "1000m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "1Gi" + volumes: + - name: vespa-certificates + secret: + secretName: vespa-certificates + items: + - key: cert.pem + path: cert.pem + - key: key.pem + path: key.pem