From 33a788acd7ca728eaeb7caedf9a7692a727cf385 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Fri, 1 Jul 2022 00:10:02 +0300 Subject: [PATCH 01/13] Add Postgres support for Tyk-Pro Signed-off-by: Burak Sekili --- tyk-pro/configs/pump-postgres.conf | 38 +++++++++++++ tyk-pro/configs/pump.conf | 28 +++++---- tyk-pro/configs/tyk_analytics.conf | 16 +----- tyk-pro/configs/tyk_mgmt.conf | 1 + tyk-pro/templates/_helpers.tpl | 26 +++++++++ tyk-pro/templates/configmap-pump.yaml | 4 +- tyk-pro/templates/deployment-dash.yaml | 76 ++++++++++++++++++------- tyk-pro/templates/deployment-pmp.yaml | 79 +++++++++++++++++++------- tyk-pro/templates/secrets.yaml | 1 + tyk-pro/values.yaml | 21 +++++-- 10 files changed, 212 insertions(+), 78 deletions(-) create mode 100644 tyk-pro/configs/pump-postgres.conf diff --git a/tyk-pro/configs/pump-postgres.conf b/tyk-pro/configs/pump-postgres.conf new file mode 100644 index 00000000..bc835540 --- /dev/null +++ b/tyk-pro/configs/pump-postgres.conf @@ -0,0 +1,38 @@ +{ + "log_level": "info", + "analytics_storage_type": "redis", + "analytics_storage_config": { + "type": "redis", + "enable_cluster": false, + "hosts": null, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000 + }, + "dont_purge_uptime_data": false, + "purge_delay": 2, + "pumps": { + "SQL": { + "type": "sql", + "meta": { + "type": "postgres", + "table_sharding": false + } + }, + "sql_aggregate": { + "name": "sql_aggregate", + "meta": { + "type": "postgres", + "table_sharding": false + } + } + }, + "uptime_pump_config": { + "uptime_type": "sql", + "type": "postgres", + "table_sharding": false, + "log_level": "info" + } +} \ No newline at end of file diff --git a/tyk-pro/configs/pump.conf b/tyk-pro/configs/pump.conf index 4dbf1bee..c1b7a23f 100644 --- a/tyk-pro/configs/pump.conf +++ b/tyk-pro/configs/pump.conf @@ -1,35 +1,33 @@ { + "log_level": "info", "analytics_storage_type": "redis", "analytics_storage_config": { "type": "redis", - "enable_cluster": false, - "hosts": {}, + "hosts": null, "username": "", - "password": "", - "database": 0, "optimisation_max_idle": 2000, "optimisation_max_active": 4000 }, - "purge_delay": 4, + "dont_purge_uptime_data": false, + "purge_delay": 2, "pumps": { - "mongo": { - "name": "mongo", + "MONGO": { + "type": "mongo", "meta": { "collection_name": "tyk_analytics", - "mongo_url": "" + "mongo_use_ssl": false } }, "mongo-pump-aggregate": { - "name": "mongo-pump-aggregate", + "type": "mongo-pump-aggregate", "meta": { - "mongo_url": "", - "use_mixed_collection": true + "use_mixed_collection": true, + "store_analytics_per_minute": false, + "track_all_paths": false } } }, "uptime_pump_config": { - "collection_name": "tyk_uptime_analytics", - "mongo_url": "" - }, - "dont_purge_uptime_data": false + "collection_name": "tyk_uptime_analytics" + } } \ No newline at end of file diff --git a/tyk-pro/configs/tyk_analytics.conf b/tyk-pro/configs/tyk_analytics.conf index 2a970845..4868dde1 100644 --- a/tyk-pro/configs/tyk_analytics.conf +++ b/tyk-pro/configs/tyk_analytics.conf @@ -1,18 +1,6 @@ { - "listen_port": 3000, - "tyk_api_config": { - "Host": "http://tyk-gateway.$namespace", - "Port": "8080", - "Secret": "$sharedsecret" - }, "enable_ownership": true, - "mongo_url": "mongodb://mongo.tyk.svc.cluster.local:27017/tyk_analytics", - "mongo_use_ssl": false, - "mongo_ssl_insecure_skip_verify": false, - "mongo_ssl_allow_invalid_hostnames": false, "page_size": 10, - "admin_secret": "$adminsecret", - "shared_node_secret": "$sharedsecret", "redis_port": 6379, "redis_host": "redis.tyk.svc.cluster.local", "notify_on_change": true, @@ -22,9 +10,7 @@ "host_config": { "enable_host_names": true, "disable_org_slug_prefix": true, - "hostname": "localhost", - "override_hostname": "dashboard-svc-tyk-pro.tyk.svc.cluster.local", - "portal_root_path": "/" + "override_hostname": "dashboard-svc-tyk-pro.tyk.svc.cluster.local" }, "home_dir": "/opt/tyk-dashboard", "use_sharded_analytics": false, diff --git a/tyk-pro/configs/tyk_mgmt.conf b/tyk-pro/configs/tyk_mgmt.conf index cf365978..89a40208 100644 --- a/tyk-pro/configs/tyk_mgmt.conf +++ b/tyk-pro/configs/tyk_mgmt.conf @@ -12,6 +12,7 @@ "tags": [] }, "app_path": "/mnt/tyk-gateway/apps", + "enable_uptime_analytics": true, "storage": { "type": "redis", "enable_cluster": false, diff --git a/tyk-pro/templates/_helpers.tpl b/tyk-pro/templates/_helpers.tpl index 1bc88708..3b672832 100644 --- a/tyk-pro/templates/_helpers.tpl +++ b/tyk-pro/templates/_helpers.tpl @@ -77,6 +77,32 @@ mongodb://mongo.{{ .Release.Namespace }}.svc.cluster.local:27017/tyk_analytics {{- end -}} {{- end -}} +{{- define "tyk-pro.pg_connection_string" -}} +{{- if .Values.postgres -}} +{{- range $key, $value := .Values.postgres }}{{ print $key "=" $value " " }}{{- end }} +{{- end -}} +{{- end -}} + +{{- define "tyk-pro.backend" -}} +{{- if .Values.backend -}} +{{- if eq "postgres" .Values.backend -}} +postgres +{{- else if eq "mongo" .Values.backend -}} +mongo +{{- end -}} +{{- else -}} +mongo +{{- end -}} +{{- end -}} + +{{- define "tyk-pro.pmp_config" -}} +{{- if eq "postgres" .Values.backend -}} +pump-postgres.conf +{{- else -}} +pump.conf +{{- end -}} +{{- end -}} + {{- /* Create Semantic Version of gateway without prefix v */}} {{- define "tyk-pro.gateway-version" -}} {{- printf "%s" .Values.gateway.image.tag | replace "v" "" -}} diff --git a/tyk-pro/templates/configmap-pump.yaml b/tyk-pro/templates/configmap-pump.yaml index eb2d4cb3..fc0d8241 100644 --- a/tyk-pro/templates/configmap-pump.yaml +++ b/tyk-pro/templates/configmap-pump.yaml @@ -9,5 +9,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - {{- (.Files.Glob "configs/pump.conf").AsConfig | nindent 2 }} + {{- $pumpConfPath := include "tyk-pro.pmp_config" . }} + {{- $pumpConfPath = printf "configs/%s" $pumpConfPath }} + {{- (.Files.Glob $pumpConfPath).AsConfig | nindent 2 }} {{- end }} diff --git a/tyk-pro/templates/deployment-dash.yaml b/tyk-pro/templates/deployment-dash.yaml index 0a898fc3..bf43825b 100644 --- a/tyk-pro/templates/deployment-dash.yaml +++ b/tyk-pro/templates/deployment-dash.yaml @@ -61,13 +61,9 @@ spec: env: - name: TYK_DB_LISTENPORT value: "{{ .Values.dash.containerPort }}" - # Lagacy support for Redis Cluster driver. Driver dropped in v3.0.0. + # Legacy support for Redis Cluster driver. Driver dropped in v3.0.0. - name: REDIGOCLUSTER_SHARDCOUNT value: "128" - - name: TYK_DB_HOSTCONFIG_HOSTNAME - value: "{{ .Values.dash.hostName }}" - - name: TYK_DB_HOSTCONFIG_GATEWAYHOSTNAME - value: "{{ .Values.gateway.hostName }}" {{- if .Values.gateway.control.enabled }} - name: TYK_DB_TYKAPI_HOST value: "{{ include "tyk-pro.gwproto" . }}://gateway-control-svc-{{ include "tyk-pro.fullname" . }}.{{ .Release.Namespace }}" @@ -79,11 +75,28 @@ spec: - name: TYK_DB_TYKAPI_PORT value: "{{ .Values.gateway.service.port }}" {{- end }} + - name: TYK_DB_TYKAPI_SECRET valueFrom: secretKeyRef: name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} key: APISecret + - name: TYK_DB_ADMINSECRET + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} + key: AdminSecret + - name: TYK_DB_NODESECRET + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} + key: APISecret + - name: TYK_DB_LICENSEKEY + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} + key: DashLicense + - name: TYK_DB_REDISADDRS value: {{ include "tyk-pro.redis_url" . | quote }} - name: TYK_DB_ENABLECLUSTER @@ -97,32 +110,51 @@ spec: value: "{{ default "false" .Values.redis.useSSL }}" - name: TYK_DB_REDISSSLINSECURESKIPVERIFY value: "true" - - name: TYK_DB_MONGOURL + + - name: TYK_DB_HTTPSERVEROPTIONS_USESSL + value: "{{ .Values.dash.tls }}" + + - name: TYK_DB_HOSTCONFIG_HOSTNAME + value: "{{ .Values.dash.hostName }}" + - name: TYK_DB_HOSTCONFIG_GATEWAYHOSTNAME + value: "{{ .Values.gateway.hostName }}" + - name: TYK_DB_HOSTCONFIG_PORTALROOTPATH + value: {{ .Values.portal.path }} + {{ if eq "postgres" (include "tyk-pro.backend" .) }} + - name: TYK_DB_STORAGE_MAIN_TYPE + value: "postgres" + - name: TYK_DB_STORAGE_MAIN_CONNECTIONSTRING valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} - key: mongoURL - - name: TYK_DB_MONGOUSESSL - value: "{{ default "false" .Values.mongo.useSSL }}" - - name: TYK_DB_ADMINSECRET + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: pgConnectionString + + - name: TYK_DB_STORAGE_ANALYTICS_TYPE + value: "postgres" + - name: TYK_DB_STORAGE_ANALYTICS_CONNECTIONSTRING valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} - key: AdminSecret - - name: TYK_DB_NODESECRET + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: pgConnectionString + + - name: TYK_DB_STORAGE_UPTIME_TYPE + value: "postgres" + - name: TYK_DB_STORAGE_UPTIME_CONNECTIONSTRING valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} - key: APISecret - - name: TYK_DB_LICENSEKEY + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: pgConnectionString + {{ else }} + - name: TYK_DB_STORAGE_MAIN_TYPE + value: "mongo" + - name: TYK_DB_STORAGE_MAIN_CONNECTIONSTRING valueFrom: secretKeyRef: name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} - key: DashLicense - - name: TYK_DB_HOSTCONFIG_PORTALROOTPATH - value: {{ .Values.portal.path }} - - name: TYK_DB_HTTPSERVEROPTIONS_USESSL - value: "{{ .Values.dash.tls }}" + key: mongoURL + - name: TYK_DB_STORAGE_MAIN_MONGO_SSL_ENABLED + value: "{{ default "false" .Values.mongo.useSSL }}" + {{ end }} {{- if .Values.dash.extraEnvs }} {{- toYaml .Values.dash.extraEnvs| nindent 10 }} {{- end }} diff --git a/tyk-pro/templates/deployment-pmp.yaml b/tyk-pro/templates/deployment-pmp.yaml index c03eaddd..98362d06 100644 --- a/tyk-pro/templates/deployment-pmp.yaml +++ b/tyk-pro/templates/deployment-pmp.yaml @@ -19,11 +19,12 @@ spec: labels: app: pump-{{ include "tyk-pro.fullname" . }} release: {{ .Release.Name }} + {{- if .Values.pump.annotations }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmap-pump.yaml") . | sha256sum }} {{- range $key, $value := .Values.pump.annotations }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- end }} spec: {{- if .Values.pump.nodeSelector }} nodeSelector: @@ -51,38 +52,76 @@ spec: drop: - all env: - # Lagacy support for Redis Cluster driver. Driver dropped in v3.0.0. + # Legacy support for Redis Cluster driver. Driver dropped in v3.0.0. - name: REDIGOCLUSTER_SHARDCOUNT value: "128" - - name: PMP_MONGO_MONGOURL + + - name: TYK_PMP_ANALYTICSSTORAGECONFIG_DATABASE + value: "{{ .Values.redis.storage.database }}" + - name: TYK_PMP_ANALYTICSSTORAGECONFIG_REDISUSESSL + value: "{{ default "false" .Values.redis.useSSL }}" + - name: TYK_PMP_ANALYTICSSTORAGECONFIG_ADDRS + value: {{ include "tyk-pro.redis_url" . | quote }} + - name: TYK_PMP_ANALYTICSSTORAGECONFIG_ENABLECLUSTER + value: "{{ default "false" .Values.redis.enableCluster }}" + - name: TYK_PMP_ANALYTICSSTORAGECONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: redisPass + + {{ if eq "postgres" (include "tyk-pro.backend" .) }} + # SQL Pump + - name: TYK_PMP_PUMPS_SQL_META_CONNECTIONSTRING + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: pgConnectionString + + # SQL Aggregate Pump + - name: TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: pgConnectionString + + # SQL Uptime Pump + - name: TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: pgConnectionString + {{ else }} + # Mongo Pump + - name: TYK_PMP_PUMPS_MONGO_META_MONGOURL + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} key: mongoURL - - name: PMP_MONGO_MONGOUSESSL + - name: TYK_PMP_PUMPS_MONGO_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" - - name: PMP_MONGOAGG_MONGOURL + + # Mongo Aggregate Pump + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} key: mongoURL - - name: PMP_MONGOAGG_MONGOUSESSL + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" - - name: TYK_PMP_ANALYTICSSTORAGECONFIG_ADDRS - value: {{ include "tyk-pro.redis_url" . | quote }} - - name: TYK_PMP_ANALYTICSSTORAGECONFIG_ENABLECLUSTER - value: "{{ default "false" .Values.redis.enableCluster }}" - - name: TYK_PMP_ANALYTICSSTORAGECONFIG_PASSWORD + + # Mongo Uptime Pump + - name: TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} - key: redisPass - - name: TYK_PMP_ANALYTICSSTORAGECONFIG_REDISUSESSL - value: "{{ default "false" .Values.redis.useSSL }}" + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end }} + key: mongoURL + {{ end }} + {{- if .Values.pump.extraEnvs }} {{- toYaml .Values.pump.extraEnvs| nindent 10 }} {{- end }} - command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/pump.conf"] + command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/{{ include "tyk-pro.pmp_config" . }}"] volumeMounts: - name: tyk-pump-conf mountPath: /etc/tyk-pump @@ -102,8 +141,8 @@ spec: configMap: name: pump-conf-{{ include "tyk-pro.fullname" . }} items: - - key: pump.conf - path: pump.conf + - key: {{ include "tyk-pro.pmp_config" . }} + path: {{ include "tyk-pro.pmp_config" . }} {{- if .Values.pump.mounts }} {{- range $secret := .Values.pump.mounts }} - name: {{ $.Release.Name }}-pump-secret-{{ $secret.name }} diff --git a/tyk-pro/templates/secrets.yaml b/tyk-pro/templates/secrets.yaml index 9a8144e5..f9aa1248 100644 --- a/tyk-pro/templates/secrets.yaml +++ b/tyk-pro/templates/secrets.yaml @@ -12,6 +12,7 @@ type: Opaque stringData: redisPass: "{{ .Values.redis.pass }}" mongoURL: {{ include "tyk-pro.mongo_url" . | quote }} + pgConnectionString: {{ include "tyk-pro.pg_connection_string" . }} AdminSecret: "{{ .Values.secrets.AdminSecret }}" APISecret: "{{ .Values.secrets.APISecret }}" DashLicense: "{{ .Values.dash.license }}" diff --git a/tyk-pro/values.yaml b/tyk-pro/values.yaml index 979d1e04..5c42ba72 100644 --- a/tyk-pro/values.yaml +++ b/tyk-pro/values.yaml @@ -41,7 +41,7 @@ redis: # Default value: redis.{{ .Release.Namespace }}.svc.cluster.local:6379 # addrs: # - redis.tyk.svc.cluster.local:6379 - # This is the the DNS name of the redis as set when using Bitnami + # This is the DNS name of the redis as set when using Bitnami # - tyk-redis-master.tyk.svc.cluster.local:6379 @@ -64,6 +64,8 @@ redis: storage: database: 0 +# Choose the backend for Tyk. [ "mongo", "postgres" ] +backend: mongo mongo: # The mongoURL value will allow you to set your MongoDB address. # Default value: mongodb://mongo.{{ .Release.Namespace }}.svc.cluster.local:27017/tyk_analytics @@ -76,8 +78,17 @@ mongo: # Default value: false useSSL: false +# Postgres connection string parameters. +postgres: + host: tyk-postgres-postgresql.tyk.svc.cluster.local + port: 5432 + user: postgres + password: + database: tyk_analytics + sslmode: + mdcb: - # Determines whither or not the mdcb component should be installed. + # Determines whither or not the MDCB component should be installed. enabled: false # If set to true the MDCB will use SSL connection. # You will also need to set the: @@ -212,7 +223,7 @@ dash: containerPort: 3000 image: repository: tykio/tyk-dashboard - tag: v3.2.1 + tag: v4.0.0 pullPolicy: Always service: type: NodePort @@ -301,7 +312,7 @@ gateway: containerPort: 8080 image: repository: tykio/tyk-gateway - tag: v3.2.1 + tag: v4.0.0 pullPolicy: Always service: type: NodePort @@ -359,7 +370,7 @@ pump: replicaCount: 1 image: repository: tykio/tyk-pump-docker-pub - tag: v1.4.0 + tag: v1.5.1 pullPolicy: Always annotations: {} resources: {} From 005b960220012531d4aaeb550e32381250c742c5 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Sat, 2 Jul 2022 15:43:36 +0300 Subject: [PATCH 02/13] Add Postgres support for Tyk-Headless Signed-off-by: Burak Sekili --- tyk-headless/configs/pump-postgres.conf | 38 ++++++++++++++ tyk-headless/configs/pump.conf | 30 +++++------ tyk-headless/configs/tyk_mgmt.conf | 3 +- tyk-headless/templates/_helpers.tpl | 29 +++++++++++ tyk-headless/templates/configmap-pump.yaml | 4 +- tyk-headless/templates/deployment-pmp.yaml | 59 +++++++++++++++++----- tyk-headless/templates/secrets.yaml | 1 + tyk-headless/values.yaml | 23 +++++---- 8 files changed, 147 insertions(+), 40 deletions(-) create mode 100644 tyk-headless/configs/pump-postgres.conf diff --git a/tyk-headless/configs/pump-postgres.conf b/tyk-headless/configs/pump-postgres.conf new file mode 100644 index 00000000..bc835540 --- /dev/null +++ b/tyk-headless/configs/pump-postgres.conf @@ -0,0 +1,38 @@ +{ + "log_level": "info", + "analytics_storage_type": "redis", + "analytics_storage_config": { + "type": "redis", + "enable_cluster": false, + "hosts": null, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000 + }, + "dont_purge_uptime_data": false, + "purge_delay": 2, + "pumps": { + "SQL": { + "type": "sql", + "meta": { + "type": "postgres", + "table_sharding": false + } + }, + "sql_aggregate": { + "name": "sql_aggregate", + "meta": { + "type": "postgres", + "table_sharding": false + } + } + }, + "uptime_pump_config": { + "uptime_type": "sql", + "type": "postgres", + "table_sharding": false, + "log_level": "info" + } +} \ No newline at end of file diff --git a/tyk-headless/configs/pump.conf b/tyk-headless/configs/pump.conf index ee1ab96d..c1b7a23f 100644 --- a/tyk-headless/configs/pump.conf +++ b/tyk-headless/configs/pump.conf @@ -1,35 +1,33 @@ { + "log_level": "info", "analytics_storage_type": "redis", "analytics_storage_config": { "type": "redis", - "enable_cluster": false, - "hosts": {}, + "hosts": null, "username": "", - "password": "", - "database": 0, "optimisation_max_idle": 2000, "optimisation_max_active": 4000 }, - "purge_delay": 4, + "dont_purge_uptime_data": false, + "purge_delay": 2, "pumps": { - "mongo": { - "name": "mongo", + "MONGO": { + "type": "mongo", "meta": { - "collection_name": "tyk_analytics_headless", - "mongo_url": "" + "collection_name": "tyk_analytics", + "mongo_use_ssl": false } }, "mongo-pump-aggregate": { - "name": "mongo-pump-aggregate", + "type": "mongo-pump-aggregate", "meta": { - "mongo_url": "", - "use_mixed_collection": true + "use_mixed_collection": true, + "store_analytics_per_minute": false, + "track_all_paths": false } } }, "uptime_pump_config": { - "collection_name": "tyk_uptime_analytics_headless", - "mongo_url": "" - }, - "dont_purge_uptime_data": false + "collection_name": "tyk_uptime_analytics" + } } \ No newline at end of file diff --git a/tyk-headless/configs/tyk_mgmt.conf b/tyk-headless/configs/tyk_mgmt.conf index e30aa92a..f83dac21 100644 --- a/tyk-headless/configs/tyk_mgmt.conf +++ b/tyk-headless/configs/tyk_mgmt.conf @@ -1,7 +1,5 @@ { "listen_port": 8080, - "secret": "$sharedsecret", - "node_secret": "$sharedsecret", "template_path": "/opt/tyk-gateway/templates", "tyk_js_path": "/opt/tyk-gateway/js/tyk.js", "middleware_path": "/mnt/tyk-gateway/middleware", @@ -12,6 +10,7 @@ "tags": [] }, "app_path": "/mnt/tyk-gateway/apps", + "enable_uptime_analytics": true, "storage": { "type": "redis", "enable_cluster": false, diff --git a/tyk-headless/templates/_helpers.tpl b/tyk-headless/templates/_helpers.tpl index 319a6794..a5dd19fc 100644 --- a/tyk-headless/templates/_helpers.tpl +++ b/tyk-headless/templates/_helpers.tpl @@ -57,3 +57,32 @@ mongodb://{{ .Values.mongo.host }}:{{ .Values.mongo.port }}/tyk_analytics mongodb://mongo.{{ .Release.Namespace }}.svc.cluster.local:27017/tyk_analytics {{- end -}} {{- end -}} + +{{- define "tyk-headless.pg_connection_string" -}} +{{- if .Values.postgres -}} +{{- range $key, $value := .Values.postgres }}{{ print $key "=" $value " " }}{{- end }} +{{- end -}} +{{- end -}} + +{{- define "tyk-headless.backend" -}} +{{- if .Values.backend -}} +{{- if eq "postgres" .Values.backend -}} +postgres +{{- else if eq "mongo" .Values.backend -}} +mongo +{{- end -}} +{{- else if .Values.mongo.enabled -}} +mongo +{{- else -}} +none +{{- end -}} +{{- end -}} + + +{{- define "tyk-headless.pmp_config" -}} +{{- if eq "postgres" .Values.backend -}} +pump-postgres.conf +{{- else -}} +pump.conf +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/tyk-headless/templates/configmap-pump.yaml b/tyk-headless/templates/configmap-pump.yaml index 2031a694..8a518c23 100644 --- a/tyk-headless/templates/configmap-pump.yaml +++ b/tyk-headless/templates/configmap-pump.yaml @@ -9,5 +9,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - {{- (.Files.Glob "configs/pump.conf").AsConfig | nindent 2 }} + {{- $pumpConfPath := include "tyk-headless.pmp_config" . }} + {{- $pumpConfPath = printf "configs/%s" $pumpConfPath }} + {{- (.Files.Glob $pumpConfPath).AsConfig | nindent 2 }} {{- end }} diff --git a/tyk-headless/templates/deployment-pmp.yaml b/tyk-headless/templates/deployment-pmp.yaml index 77a26b31..1c8b1ec1 100644 --- a/tyk-headless/templates/deployment-pmp.yaml +++ b/tyk-headless/templates/deployment-pmp.yaml @@ -48,22 +48,57 @@ spec: drop: - all env: - # Lagacy support for Redis Cluster driver. Driver dropped in v3.0.0. + # Legacy support for Redis Cluster driver. Driver dropped in v3.0.0. - name: REDIGOCLUSTER_SHARDCOUNT value: "128" - {{- if .Values.mongo.enabled }} - - name: PMP_MONGO_MONGOURL - value: {{ include "tyk-headless.mongo_url" . | quote }} - - name: PMP_MONGO_MONGOUSESSL + {{- if eq "postgres" (include "tyk-headless.backend" .) }} + # SQL Pump + - name: TYK_PMP_PUMPS_SQL_META_CONNECTIONSTRING + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} + key: pgConnectionString + + # SQL Aggregate Pump + - name: TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} + key: pgConnectionString + + # SQL Uptime Pump + - name: TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} + key: pgConnectionString + {{ else }} + # Mongo Pump + - name: TYK_PMP_PUMPS_MONGO_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" - - name: PMP_MONGOAGG_MONGOURL + - name: TYK_PMP_PUMPS_MONGO_META_MONGOURL valueFrom: secretKeyRef: - name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end}} + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} key: mongoURL - - name: PMP_MONGOAGG_MONGOUSESSL + + # Mongo Aggregate Pump + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" - {{- end }} + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} + key: mongoURL + + # Mongo Uptime Pump + - name: TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} + key: mongoURL + {{ end }} + - name: TYK_PMP_ANALYTICSSTORAGECONFIG_ADDRS {{- if and .Values.redis.host .Values.redis.port }} value: "{{ .Values.redis.host }}:{{ .Values.redis.port }}" @@ -100,7 +135,7 @@ spec: value: {{ $env.value | quote }} {{- end }} {{- end }} - command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/pump.conf"] + command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/{{ include "tyk-headless.pmp_config" . }}"] volumeMounts: - name: tyk-pump-conf mountPath: /etc/tyk-pump @@ -120,8 +155,8 @@ spec: configMap: name: pump-conf-{{ include "tyk-headless.fullname" . }} items: - - key: pump.conf - path: pump.conf + - key: {{ include "tyk-headless.pmp_config" . }} + path: {{ include "tyk-headless.pmp_config" . }} {{- if .Values.pump.mounts }} {{- range $secret := .Values.pump.mounts }} - name: {{ $.Release.Name }}-pump-secret-{{ $secret.name }} diff --git a/tyk-headless/templates/secrets.yaml b/tyk-headless/templates/secrets.yaml index b499a124..3111b7c4 100644 --- a/tyk-headless/templates/secrets.yaml +++ b/tyk-headless/templates/secrets.yaml @@ -13,5 +13,6 @@ stringData: redisPass: "{{ .Values.redis.pass }}" redisSentinelPass: "{{ .Values.redis.sentinelPass }}" mongoURL: {{ include "tyk-headless.mongo_url" . | quote }} + pgConnectionString: {{ include "tyk-headless.pg_connection_string" . }} APISecret: "{{ .Values.secrets.APISecret }}" {{- end }} diff --git a/tyk-headless/values.yaml b/tyk-headless/values.yaml index 635eee8b..961ecd8c 100644 --- a/tyk-headless/values.yaml +++ b/tyk-headless/values.yaml @@ -60,13 +60,9 @@ redis: storage: database: 0 +# Choose the pumps backend for Tyk. [ "", "mongo", "postgres" ] +backend: "" mongo: - # If you want to collect analytics through the mongo pumps you can turn this - # option on. Once on, Tyk Pump will assume that MongoDB is avaibale at - # mongo.tyk.svc.cluster.local:27017 if it is not, please change the mongoURL - # value below. - enabled: false - # The mongoURL value will allow you to set your MongoDB address. # Default value: mongodb://mongo.{{ .Release.Namespace }}.svc.cluster.local:27017/tyk_analytics # mongoURL: mongodb://mongo.tyk.svc.cluster.local:27017/tyk_analytics @@ -75,7 +71,16 @@ mongo: # Enables SSL for MongoDB connection. MongoDB instance will have to support that. # Default value: false - # useSSL: true + useSSL: false + +# Postgres connection string parameters. +postgres: + host: tyk-postgres-postgresql.tyk.svc.cluster.local + port: 5432 + user: postgres + password: + database: tyk_analytics + sslmode: gateway: # The hostname to bind the Gateway to. @@ -88,7 +93,7 @@ gateway: containerPort: 8080 image: repository: docker.tyk.io/tyk-gateway/tyk-gateway - tag: v3.2.1 + tag: v4.0.0 pullPolicy: IfNotPresent service: type: NodePort @@ -151,7 +156,7 @@ pump: replicaCount: 1 image: repository: docker.tyk.io/tyk-pump/tyk-pump - tag: v1.4.0 + tag: v1.5.1 pullPolicy: IfNotPresent annotations: {} resources: {} From 9b0fa28725d4fad7821c3f804bb25bbc60e91e6f Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Sun, 3 Jul 2022 17:33:49 +0300 Subject: [PATCH 03/13] Update Gateway version in hybrid Signed-off-by: Burak Sekili --- tyk-hybrid/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyk-hybrid/values.yaml b/tyk-hybrid/values.yaml index b4d6277a..11a3e67e 100644 --- a/tyk-hybrid/values.yaml +++ b/tyk-hybrid/values.yaml @@ -96,7 +96,7 @@ gateway: containerPort: 8080 image: repository: tykio/tyk-gateway - tag: v3.2.1 + tag: v4.0.0 pullPolicy: Always service: type: NodePort From bf94f2baabb6c05d7e642cd494146a0f4d28979f Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Sun, 3 Jul 2022 17:34:27 +0300 Subject: [PATCH 04/13] Add postgres instructions in main Readme Signed-off-by: Burak Sekili --- README.md | 53 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 758cd5d9..5a4a7027 100644 --- a/README.md +++ b/README.md @@ -4,31 +4,54 @@ Tyk provides 3 different helm charts in this repo. Please visit the respective p - [Tyk Hybrid](/~https://github.com/TykTechnologies/tyk-helm-chart/tree/master/tyk-hybrid) - [Tyk Headless](/~https://github.com/TykTechnologies/tyk-helm-chart/tree/master/tyk-headless) -## Redis and MongoDB +## Redis and MongoDB or PostgreSQL - Redis is required for all of the Tyk installations it must be installed in the cluster or reachable from inside K8s. -- MongoDB is only required for the tyk-pro installation and must be installed in the cluster, or reachable from inside K8s. If you are using the mongo pumps in the tyk-headless installation you will require mongo installed for that as well. +- MongoDB or PostgreSQL are only required for the tyk-pro installation and must be installed in the cluster, or reachable from inside K8s. If you are using the MongoDB or SQL pumps in the tyk-headless installation you will require MongoDB or PostgreSQL installed for that as well. -For Redis and MongoDB you can use these rather excellent charts provided by Bitnami: +For Redis you can use these rather excellent charts provided by Bitnami: +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +kubectl create namespace tyk - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update - kubectl create namespace tyk - helm install tyk-mongo bitnami/mongodb --set "replicaSet.enabled=true" -n tyk - (follow notes from the installation output to get connection details and update them in `values.yaml` file) - helm install tyk-redis bitnami/redis -n tyk - (follow notes from the installation output to get connection details and update them in `values.yaml` file) +helm install tyk-redis bitnami/redis -n tyk +(follow notes from the installation output to get connection details and update them in `values.yaml` file) +``` + +For Mongo or PostgreSQL you can use these rather excellent charts provided by Bitnami: +```bash +helm install tyk-mongo bitnami/mongodb --version {HELM_CHART_VERSION} --set "replicaSet.enabled=true" -n tyk +(follow notes from the installation output to get connection details and update them in `values.yaml` file) +``` + +>**_NOTE_**: [Here](https://tyk.io/docs/planning-for-production/redis-mongodb/#supported-versions) is list of supported mongo versions. Please make sure you are installing mongo helm chart that matches these version. + +```bash +helm install tyk-postgres bitnami/postgresql --set "auth.database=tyk_analytics" -n tyk +(follow notes from the installation output to get connection details and update them in `values.yaml` file) +``` *Important Note regarding MongoDB:* This helm chart enables the PodDisruptionBudget for MongoDB with an arbiter replica-count of 1. If you intend to perform system maintenance on the node where the MongoDB pod is running and this maintenance requires for the node to be drained, this action will be prevented due the replica count being 1. Increase the replica count in the helm chart deployment to a minimum of 2 to remedy this issue. Another option for Redis and MongoDB, if you want to get started quickly is to use our simple charts. **Please note that these provided charts must not ever be used in production and for anything but a quick start evaluation only, use external DBs or Official Helm charts for MongoDB and Redis in any other case.** We provide these charts so you can quickly have Tyk running however they are not meant for long term storage of data for example. - kubectl create namespace tyk - helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ - helm repo update - helm install redis tyk-helm/simple-redis -n tyk - helm install mongo tyk-helm/simple-mongodb -n tyk +```bash +kubectl create namespace tyk +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update +``` + +#### Redis + +```bash +helm install redis tyk-helm/simple-redis -n tyk +``` +#### MongoDB +```bash +helm install mongo tyk-helm/simple-mongodb -n tyk +``` ## TLS You can turn on the tls option under the gateway section in the values.yaml files which will make the gateways listen on port 443 and load up a dummy certificate. You can set your own default certificate by replacing the files in the certs/ folder. From a126338ab0c3144c7347f22bdadb09a7e8878645 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Sun, 3 Jul 2022 17:36:48 +0300 Subject: [PATCH 05/13] Add pump installation instructions in tyk-headless Readme Signed-off-by: Burak Sekili --- tyk-headless/README.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tyk-headless/README.md b/tyk-headless/README.md index 150c168c..3c4bc6c6 100644 --- a/tyk-headless/README.md +++ b/tyk-headless/README.md @@ -1,5 +1,5 @@ ## Tyk OSS Gateway -This chart deploys the open source Tyk Gateway. +This chart deploys the open source Tyk Gateway and Pump. --- @@ -13,8 +13,26 @@ Redis should already be installed or accessible by the gateway. Check [Tyk OSS Helm chart Installation](https://tyk.io/docs/tyk-oss/ce-helm-chart/) for the full details. ## Installation +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update +kubectl create namespace tyk +helm install tyk-headless tyk-helm/tyk-headless -n tyk +``` - helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ - helm repo update - kubectl create namespace tyk - helm install tyk-ce tyk-helm/tyk-headless -n tyk --create-namespace +## Pump Installation +By default pump installation is disabled. You can enable it by setting `pump.enabled` to `true` in `values.yaml` file. +Alternatively, you can use `--set pump.enabled=true` while doing helm install. + +## Pump Configuration +### Mongo Pump + +To configure mongo pump, do following changings in `values.yaml` file: +1. Set `backend` to `mongo`. +2. Set connection string in `mongo.mongoURL`. + + +### Postgres Pump +To configure postgres pump, do following changings in `values.yaml` file: +1. Set `backend` to `postgres`. +2. Set connection string parameters in `postgres` section. From 24f788756642f0d289e37a310b1ed92d654bd46d Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Sun, 3 Jul 2022 17:37:32 +0300 Subject: [PATCH 06/13] Add Postgres instructions in tyk-pro Readme Signed-off-by: Burak Sekili --- tyk-pro/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyk-pro/README.md b/tyk-pro/README.md index b3f9d2a3..d4838321 100644 --- a/tyk-pro/README.md +++ b/tyk-pro/README.md @@ -9,7 +9,7 @@ and MDCB (Multi Data-centre Bridge). --- ## Prerequisites -Redis and MongoDB should already be installed or accessible by the gateway. +Redis and MongoDB or PostgreSQL should already be installed or accessible by the gateway. Check [Tyk Self-managed Helm chart Installation](https://tyk.io/docs/tyk-self-managed/tyk-helm-chart/) for the full details. From 57386ba543b04a5d1a2301e850be2d2a689cf981 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Mon, 4 Jul 2022 10:13:57 +0300 Subject: [PATCH 07/13] Resolve trailing whitespace issue in postgres.password field of headless' values file Signed-off-by: Burak Sekili --- tyk-headless/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyk-headless/values.yaml b/tyk-headless/values.yaml index 961ecd8c..24f0ba2a 100644 --- a/tyk-headless/values.yaml +++ b/tyk-headless/values.yaml @@ -78,7 +78,7 @@ postgres: host: tyk-postgres-postgresql.tyk.svc.cluster.local port: 5432 user: postgres - password: + password: database: tyk_analytics sslmode: From 689e5c88fa0425e4f9ed1dab17deac06cdc17ad8 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Mon, 4 Jul 2022 10:22:15 +0300 Subject: [PATCH 08/13] Resolve trailing whitespace issue in postgres.password field of pro values file Signed-off-by: Burak Sekili --- tyk-pro/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyk-pro/values.yaml b/tyk-pro/values.yaml index 5c42ba72..faf5b034 100644 --- a/tyk-pro/values.yaml +++ b/tyk-pro/values.yaml @@ -83,7 +83,7 @@ postgres: host: tyk-postgres-postgresql.tyk.svc.cluster.local port: 5432 user: postgres - password: + password: database: tyk_analytics sslmode: From b98e5be6375d4bfd45fa8f51657a2975d5d0cff0 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Thu, 7 Jul 2022 16:19:22 +0300 Subject: [PATCH 09/13] Remove extra config file for postgres Signed-off-by: Burak Sekili --- tyk-pro/configs/pump.conf | 17 ----------------- tyk-pro/templates/_helpers.tpl | 7 ------- tyk-pro/templates/configmap-pump.yaml | 4 +--- tyk-pro/templates/deployment-dash.yaml | 4 +++- tyk-pro/templates/deployment-pmp.yaml | 26 +++++++++++++++++++++++--- 5 files changed, 27 insertions(+), 31 deletions(-) diff --git a/tyk-pro/configs/pump.conf b/tyk-pro/configs/pump.conf index c1b7a23f..2ade99e2 100644 --- a/tyk-pro/configs/pump.conf +++ b/tyk-pro/configs/pump.conf @@ -10,23 +10,6 @@ }, "dont_purge_uptime_data": false, "purge_delay": 2, - "pumps": { - "MONGO": { - "type": "mongo", - "meta": { - "collection_name": "tyk_analytics", - "mongo_use_ssl": false - } - }, - "mongo-pump-aggregate": { - "type": "mongo-pump-aggregate", - "meta": { - "use_mixed_collection": true, - "store_analytics_per_minute": false, - "track_all_paths": false - } - } - }, "uptime_pump_config": { "collection_name": "tyk_uptime_analytics" } diff --git a/tyk-pro/templates/_helpers.tpl b/tyk-pro/templates/_helpers.tpl index 3b672832..6d422718 100644 --- a/tyk-pro/templates/_helpers.tpl +++ b/tyk-pro/templates/_helpers.tpl @@ -95,13 +95,6 @@ mongo {{- end -}} {{- end -}} -{{- define "tyk-pro.pmp_config" -}} -{{- if eq "postgres" .Values.backend -}} -pump-postgres.conf -{{- else -}} -pump.conf -{{- end -}} -{{- end -}} {{- /* Create Semantic Version of gateway without prefix v */}} {{- define "tyk-pro.gateway-version" -}} diff --git a/tyk-pro/templates/configmap-pump.yaml b/tyk-pro/templates/configmap-pump.yaml index fc0d8241..eb2d4cb3 100644 --- a/tyk-pro/templates/configmap-pump.yaml +++ b/tyk-pro/templates/configmap-pump.yaml @@ -9,7 +9,5 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - {{- $pumpConfPath := include "tyk-pro.pmp_config" . }} - {{- $pumpConfPath = printf "configs/%s" $pumpConfPath }} - {{- (.Files.Glob $pumpConfPath).AsConfig | nindent 2 }} + {{- (.Files.Glob "configs/pump.conf").AsConfig | nindent 2 }} {{- end }} diff --git a/tyk-pro/templates/deployment-dash.yaml b/tyk-pro/templates/deployment-dash.yaml index bf43825b..75475170 100644 --- a/tyk-pro/templates/deployment-dash.yaml +++ b/tyk-pro/templates/deployment-dash.yaml @@ -75,12 +75,12 @@ spec: - name: TYK_DB_TYKAPI_PORT value: "{{ .Values.gateway.service.port }}" {{- end }} - - name: TYK_DB_TYKAPI_SECRET valueFrom: secretKeyRef: name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-pro.fullname" . }} {{ end}} key: APISecret + - name: TYK_DB_ADMINSECRET valueFrom: secretKeyRef: @@ -121,6 +121,8 @@ spec: - name: TYK_DB_HOSTCONFIG_PORTALROOTPATH value: {{ .Values.portal.path }} {{ if eq "postgres" (include "tyk-pro.backend" .) }} + - name: TYK_DB_ENABLEAGGREGATELOOKUPS + value: "false" - name: TYK_DB_STORAGE_MAIN_TYPE value: "postgres" - name: TYK_DB_STORAGE_MAIN_CONNECTIONSTRING diff --git a/tyk-pro/templates/deployment-pmp.yaml b/tyk-pro/templates/deployment-pmp.yaml index 98362d06..e70dc0df 100644 --- a/tyk-pro/templates/deployment-pmp.yaml +++ b/tyk-pro/templates/deployment-pmp.yaml @@ -72,6 +72,10 @@ spec: {{ if eq "postgres" (include "tyk-pro.backend" .) }} # SQL Pump + - name: TYK_PMP_PUMPS_SQL_TYPE + value: "sql" + - name: TYK_PMP_PUMPS_SQL_META_TYPE + value: "postgres" - name: TYK_PMP_PUMPS_SQL_META_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -79,6 +83,10 @@ spec: key: pgConnectionString # SQL Aggregate Pump + - name: TYK_PMP_PUMPS_SQLAGGREGATE_TYPE + value: "sql_aggregate" + - name: TYK_PMP_PUMPS_SQLAGGREGATE_META_TYPE + value: "postgres" - name: TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -86,6 +94,10 @@ spec: key: pgConnectionString # SQL Uptime Pump + - name: TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE + value: "sql" + - name: TYK_PMP_UPTIMEPUMPCONFIG_TYPE + value: "postgres" - name: TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -100,8 +112,12 @@ spec: key: mongoURL - name: TYK_PMP_PUMPS_MONGO_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" + - name: TYK_PMP_PUMPS_MONGO_META_COLLECTIONNAME + value: "tyk_analytics" # Mongo Aggregate Pump + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE + value: "mongo-pump-aggregate" - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL valueFrom: secretKeyRef: @@ -109,6 +125,10 @@ spec: key: mongoURL - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_COLLECTIONNAME + value: "tyk_analytics" + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_USEMIXEDCOLLECTION + value: "true" # Mongo Uptime Pump - name: TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL @@ -121,7 +141,7 @@ spec: {{- if .Values.pump.extraEnvs }} {{- toYaml .Values.pump.extraEnvs| nindent 10 }} {{- end }} - command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/{{ include "tyk-pro.pmp_config" . }}"] + command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/pump.conf"] volumeMounts: - name: tyk-pump-conf mountPath: /etc/tyk-pump @@ -141,8 +161,8 @@ spec: configMap: name: pump-conf-{{ include "tyk-pro.fullname" . }} items: - - key: {{ include "tyk-pro.pmp_config" . }} - path: {{ include "tyk-pro.pmp_config" . }} + - key: pump.conf + path: pump.conf {{- if .Values.pump.mounts }} {{- range $secret := .Values.pump.mounts }} - name: {{ $.Release.Name }}-pump-secret-{{ $secret.name }} From 4502e12325ca3f6647840eac148ca49cd5163530 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Thu, 7 Jul 2022 16:27:15 +0300 Subject: [PATCH 10/13] remove pump-postgres.conf Signed-off-by: Burak Sekili --- tyk-pro/configs/pump-postgres.conf | 38 ------------------------------ 1 file changed, 38 deletions(-) delete mode 100644 tyk-pro/configs/pump-postgres.conf diff --git a/tyk-pro/configs/pump-postgres.conf b/tyk-pro/configs/pump-postgres.conf deleted file mode 100644 index bc835540..00000000 --- a/tyk-pro/configs/pump-postgres.conf +++ /dev/null @@ -1,38 +0,0 @@ -{ - "log_level": "info", - "analytics_storage_type": "redis", - "analytics_storage_config": { - "type": "redis", - "enable_cluster": false, - "hosts": null, - "username": "", - "password": "", - "database": 0, - "optimisation_max_idle": 2000, - "optimisation_max_active": 4000 - }, - "dont_purge_uptime_data": false, - "purge_delay": 2, - "pumps": { - "SQL": { - "type": "sql", - "meta": { - "type": "postgres", - "table_sharding": false - } - }, - "sql_aggregate": { - "name": "sql_aggregate", - "meta": { - "type": "postgres", - "table_sharding": false - } - } - }, - "uptime_pump_config": { - "uptime_type": "sql", - "type": "postgres", - "table_sharding": false, - "log_level": "info" - } -} \ No newline at end of file From 689a9cc6352c4205b53b7c1a45ad4e54a7784a0f Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Thu, 7 Jul 2022 17:25:48 +0300 Subject: [PATCH 11/13] Remove extra pump config file for postgres in headless Signed-off-by: Burak Sekili --- tyk-headless/configs/pump-postgres.conf | 38 ---------------------- tyk-headless/configs/pump.conf | 21 ++---------- tyk-headless/configs/tyk_mgmt.conf | 1 - tyk-headless/templates/_helpers.tpl | 9 ----- tyk-headless/templates/configmap-pump.yaml | 4 +-- tyk-headless/templates/deployment-pmp.yaml | 26 +++++++++++++-- tyk-headless/values.yaml | 2 +- tyk-pro/configs/pump.conf | 2 +- 8 files changed, 28 insertions(+), 75 deletions(-) delete mode 100644 tyk-headless/configs/pump-postgres.conf diff --git a/tyk-headless/configs/pump-postgres.conf b/tyk-headless/configs/pump-postgres.conf deleted file mode 100644 index bc835540..00000000 --- a/tyk-headless/configs/pump-postgres.conf +++ /dev/null @@ -1,38 +0,0 @@ -{ - "log_level": "info", - "analytics_storage_type": "redis", - "analytics_storage_config": { - "type": "redis", - "enable_cluster": false, - "hosts": null, - "username": "", - "password": "", - "database": 0, - "optimisation_max_idle": 2000, - "optimisation_max_active": 4000 - }, - "dont_purge_uptime_data": false, - "purge_delay": 2, - "pumps": { - "SQL": { - "type": "sql", - "meta": { - "type": "postgres", - "table_sharding": false - } - }, - "sql_aggregate": { - "name": "sql_aggregate", - "meta": { - "type": "postgres", - "table_sharding": false - } - } - }, - "uptime_pump_config": { - "uptime_type": "sql", - "type": "postgres", - "table_sharding": false, - "log_level": "info" - } -} \ No newline at end of file diff --git a/tyk-headless/configs/pump.conf b/tyk-headless/configs/pump.conf index c1b7a23f..aa48fdd6 100644 --- a/tyk-headless/configs/pump.conf +++ b/tyk-headless/configs/pump.conf @@ -10,24 +10,7 @@ }, "dont_purge_uptime_data": false, "purge_delay": 2, - "pumps": { - "MONGO": { - "type": "mongo", - "meta": { - "collection_name": "tyk_analytics", - "mongo_use_ssl": false - } - }, - "mongo-pump-aggregate": { - "type": "mongo-pump-aggregate", - "meta": { - "use_mixed_collection": true, - "store_analytics_per_minute": false, - "track_all_paths": false - } - } - }, "uptime_pump_config": { - "collection_name": "tyk_uptime_analytics" + "collection_name": "tyk_uptime_analytics_headless" } -} \ No newline at end of file +} diff --git a/tyk-headless/configs/tyk_mgmt.conf b/tyk-headless/configs/tyk_mgmt.conf index f83dac21..769e80cd 100644 --- a/tyk-headless/configs/tyk_mgmt.conf +++ b/tyk-headless/configs/tyk_mgmt.conf @@ -10,7 +10,6 @@ "tags": [] }, "app_path": "/mnt/tyk-gateway/apps", - "enable_uptime_analytics": true, "storage": { "type": "redis", "enable_cluster": false, diff --git a/tyk-headless/templates/_helpers.tpl b/tyk-headless/templates/_helpers.tpl index a5dd19fc..5641f9bd 100644 --- a/tyk-headless/templates/_helpers.tpl +++ b/tyk-headless/templates/_helpers.tpl @@ -77,12 +77,3 @@ mongo none {{- end -}} {{- end -}} - - -{{- define "tyk-headless.pmp_config" -}} -{{- if eq "postgres" .Values.backend -}} -pump-postgres.conf -{{- else -}} -pump.conf -{{- end -}} -{{- end -}} \ No newline at end of file diff --git a/tyk-headless/templates/configmap-pump.yaml b/tyk-headless/templates/configmap-pump.yaml index 8a518c23..2031a694 100644 --- a/tyk-headless/templates/configmap-pump.yaml +++ b/tyk-headless/templates/configmap-pump.yaml @@ -9,7 +9,5 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - {{- $pumpConfPath := include "tyk-headless.pmp_config" . }} - {{- $pumpConfPath = printf "configs/%s" $pumpConfPath }} - {{- (.Files.Glob $pumpConfPath).AsConfig | nindent 2 }} + {{- (.Files.Glob "configs/pump.conf").AsConfig | nindent 2 }} {{- end }} diff --git a/tyk-headless/templates/deployment-pmp.yaml b/tyk-headless/templates/deployment-pmp.yaml index 1c8b1ec1..50c3334f 100644 --- a/tyk-headless/templates/deployment-pmp.yaml +++ b/tyk-headless/templates/deployment-pmp.yaml @@ -53,6 +53,10 @@ spec: value: "128" {{- if eq "postgres" (include "tyk-headless.backend" .) }} # SQL Pump + - name: TYK_PMP_PUMPS_SQL_TYPE + value: "sql" + - name: TYK_PMP_PUMPS_SQL_META_TYPE + value: "postgres" - name: TYK_PMP_PUMPS_SQL_META_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -60,6 +64,10 @@ spec: key: pgConnectionString # SQL Aggregate Pump + - name: TYK_PMP_PUMPS_SQLAGGREGATE_TYPE + value: "sql_aggregate" + - name: TYK_PMP_PUMPS_SQLAGGREGATE_META_TYPE + value: "postgres" - name: TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -67,6 +75,10 @@ spec: key: pgConnectionString # SQL Uptime Pump + - name: TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE + value: "sql" + - name: TYK_PMP_UPTIMEPUMPCONFIG_TYPE + value: "postgres" - name: TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -74,6 +86,8 @@ spec: key: pgConnectionString {{ else }} # Mongo Pump + - name: TYK_PMP_PUMPS_MONGO_META_COLLECTIONNAME + value: "tyk_analytics_headless" - name: TYK_PMP_PUMPS_MONGO_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" - name: TYK_PMP_PUMPS_MONGO_META_MONGOURL @@ -83,6 +97,8 @@ spec: key: mongoURL # Mongo Aggregate Pump + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE + value: "mongo-pump-aggregate" - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOUSESSL value: "{{ default "false" .Values.mongo.useSSL }}" - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL @@ -90,6 +106,10 @@ spec: secretKeyRef: name: {{ if .Values.secrets.useSecretName }} {{ .Values.secrets.useSecretName }} {{ else }} secrets-{{ include "tyk-headless.fullname" . }} {{ end }} key: mongoURL + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_COLLECTIONNAME + value: "tyk_analytics_headless" + - name: TYK_PMP_PUMPS_MONGOAGGREGATE_META_USEMIXEDCOLLECTION + value: "true" # Mongo Uptime Pump - name: TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL @@ -135,7 +155,7 @@ spec: value: {{ $env.value | quote }} {{- end }} {{- end }} - command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/{{ include "tyk-headless.pmp_config" . }}"] + command: ["/opt/tyk-pump/tyk-pump", "-c", "/etc/tyk-pump/pump.conf"] volumeMounts: - name: tyk-pump-conf mountPath: /etc/tyk-pump @@ -155,8 +175,8 @@ spec: configMap: name: pump-conf-{{ include "tyk-headless.fullname" . }} items: - - key: {{ include "tyk-headless.pmp_config" . }} - path: {{ include "tyk-headless.pmp_config" . }} + - key: pump.conf + path: pump.conf {{- if .Values.pump.mounts }} {{- range $secret := .Values.pump.mounts }} - name: {{ $.Release.Name }}-pump-secret-{{ $secret.name }} diff --git a/tyk-headless/values.yaml b/tyk-headless/values.yaml index 24f0ba2a..552a7c58 100644 --- a/tyk-headless/values.yaml +++ b/tyk-headless/values.yaml @@ -150,7 +150,7 @@ gateway: # possible pump configs can be found here: # /~https://github.com/TykTechnologies/tyk-pump#configuration pump: - # Determines whither or not the pump component should be installed. + # Determines whether or not the pump component should be installed. enabled: false replicaCount: 1 diff --git a/tyk-pro/configs/pump.conf b/tyk-pro/configs/pump.conf index 2ade99e2..83db8db4 100644 --- a/tyk-pro/configs/pump.conf +++ b/tyk-pro/configs/pump.conf @@ -13,4 +13,4 @@ "uptime_pump_config": { "collection_name": "tyk_uptime_analytics" } -} \ No newline at end of file +} From ebd01fafd916571cff2ae95c7a591cba2cbd5ab5 Mon Sep 17 00:00:00 2001 From: Burak Sekili Date: Thu, 7 Jul 2022 17:28:59 +0300 Subject: [PATCH 12/13] update comments in values files of hybrid and values based on feedback Signed-off-by: Burak Sekili --- tyk-hybrid/values.yaml | 2 +- tyk-pro/values.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tyk-hybrid/values.yaml b/tyk-hybrid/values.yaml index 11a3e67e..4cabf58f 100644 --- a/tyk-hybrid/values.yaml +++ b/tyk-hybrid/values.yaml @@ -42,7 +42,7 @@ redis: # Default value: false # useSSL: true - # The enableCluster value will allow you to indicate to Tyk whither you are running + # The enableCluster value will allow you to indicate to Tyk whether you are running # a Redis cluster or not. # Default value: false # enableCluster: true diff --git a/tyk-pro/values.yaml b/tyk-pro/values.yaml index faf5b034..d0212052 100644 --- a/tyk-pro/values.yaml +++ b/tyk-pro/values.yaml @@ -53,7 +53,7 @@ redis: # Default value: false # useSSL: true - # The enableCluster value will allow you to indicate to Tyk whither you are + # The enableCluster value will allow you to indicate to Tyk whether you are # running a Redis cluster or not. # Default value: false # enableCluster: true @@ -88,7 +88,7 @@ postgres: sslmode: mdcb: - # Determines whither or not the MDCB component should be installed. + # Determines whether or not the MDCB component should be installed. enabled: false # If set to true the MDCB will use SSL connection. # You will also need to set the: @@ -134,7 +134,7 @@ mdcb: mounts: [] tib: - # Determines whither or not the tib component should be installed. + # Determines whether or not the tib component should be installed. enabled: false # If set to true TIB will use SSL connection. THIS IS RECOMMENDED. # You will also need to set the: @@ -192,7 +192,7 @@ tib: mounts: [] dash: - # Determines whither or not the dashboard component should be installed. + # Determines whether or not the dashboard component should be installed. enabled: true # Dashboard will only bootstrap if the master bootstrap option is set to true. bootstrap: true @@ -289,7 +289,7 @@ portal: # hosts: # - chart-example.local gateway: - # Determines whither or not the gateway component should be installed. + # Determines whether or not the gateway component should be installed. enabled: true # The hostname to bind the Gateway to. hostName: tyk-gw.local @@ -364,7 +364,7 @@ gateway: mounts: [] pump: - # Determines whither or not the pump component should be installed. + # Determines whether or not the pump component should be installed. enabled: true replicaCount: 1 From 5bb1cdb7c7ffc6bc3ae1230c372915b650c2e218 Mon Sep 17 00:00:00 2001 From: Komal Sukhani Date: Tue, 12 Jul 2022 16:32:38 +0530 Subject: [PATCH 13/13] Add back annotation for pump config hash --- tyk-pro/templates/deployment-pmp.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tyk-pro/templates/deployment-pmp.yaml b/tyk-pro/templates/deployment-pmp.yaml index e70dc0df..a0c450b2 100644 --- a/tyk-pro/templates/deployment-pmp.yaml +++ b/tyk-pro/templates/deployment-pmp.yaml @@ -19,8 +19,9 @@ spec: labels: app: pump-{{ include "tyk-pro.fullname" . }} release: {{ .Release.Name }} - {{- if .Values.pump.annotations }} annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-pump.yaml") . | sha256sum }} + {{- if .Values.pump.annotations }} {{- range $key, $value := .Values.pump.annotations }} {{ $key }}: {{ $value | quote }} {{- end }}