Merge pull request #1 from allianz-direct/feat/add-containerd-support

feat: add containerd helm support
allianz-direct · Sep 23, 2022 · 35cf476 · 35cf476
2 parents 1a4daa5 + 01d7573
commit 35cf476
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -17,6 +17,8 @@ Usage of ./tekton-s3-log-reader:
     	Bucket name or S3_BUCKET_NAME env var
   -cert string
     	TLS Certificate
+  -containerd
+    	Use containerd log format
   -key string
     	TLS Key
   -p string
@@ -32,6 +34,7 @@ tekton-s3-log-reader has `/metrics` endpoint to monitor the behaviour using Prom
 
 ## Sample stack configuration
 ### FluentBit Configuration
+#### Docker
 ```yaml
 customParsers: |
     [PARSER]
@@ -70,14 +73,65 @@ outputs: |
         Name            s3
         Alias           s3_tekton_logs
         Match           kube.tekton.*
-        bucket          eks-infra-tekton-logs
+        bucket          YOUR_BUCKER
+        region          eu-central-1
+        total_file_size 250M
+        upload_timeout  1m
+        s3_key_format   /$TAG[2]/$TAG[3]/$TAG[4]/%Y%m%d%H%M%S.log
+        s3_key_format_tag_delimiters .
+```
+#### Containerd
+```yaml
+customParsers: |
+    [PARSER]
+        # http://rubular.com/r/tjUt3Awgg4
+        Name cri-custom
+        Format regex
+        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+filters: |
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+        Buffer_Size 64K
+        # Merge_Log_Key    log_processed
+    [FILTER]
+        Name record_modifier
+        Match kube.*
+        Remove_key logtag
+        Remove_key stream
+inputs: |
+    [INPUT]
+        Name              tail
+        Alias             tekton-semantic
+        Path              /var/log/containers/*_build-release_*
+        Parser            cri-custom
+        Tag               kube.tekton.<namespace_name>.<pod_name>.<container_name>
+        Tag_Regex         (?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-
+        Mem_Buf_Limit     100MB
+        Refresh_Interval  60
+        #DB                /fluentbit/db/tail.tekton.db
+        # the database is accessed only by Fluent Bit
+        #DB.locking        True
+        # Skip_Long_Lines On
+outputs: |
+    
+    [OUTPUT]
+        Name            s3
+        Alias           s3_tekton_logs
+        Match           kube.tekton.*
+        bucket          YOUR_BUCKER
         region          eu-central-1
         total_file_size 250M
         upload_timeout  1m
         s3_key_format   /$TAG[2]/$TAG[3]/$TAG[4]/%Y%m%d%H%M%S.log
         s3_key_format_tag_delimiters .
 ```
-
 ### Tekton Dashboard Configuration
 
 Add the following flag pointing to the endpoint `tekton-s3-log-reader`: 

diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml
@@ -19,6 +19,7 @@ spec:
       {{- end }}
       labels:
         {{- include "tekton-s3-log-reader.selectorLabels" . | nindent 8 }}
+        {{- toYaml .Values.labels | nindent 8 }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -36,6 +37,10 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if .Values.args }}
+          args:
+            {{- toYaml .Values.args | nindent 12 }}
+          {{- end }}
           ports:
             - name: exporter
               containerPort: {{ .Values.service.port }}

diff --git a/main.go b/main.go
@@ -70,6 +70,7 @@ func main() {
 	tls := flag.Bool("tls", false, "Use TLS to expose endpoint")
 	serverCert := flag.String("cert", "", "TLS Certificate")
 	serverKey := flag.String("key", "", "TLS Key")
+	containerd := flag.Bool("containerd", false, "Use containerd log format")
 
 	flag.Parse()
 
@@ -93,7 +94,12 @@ func main() {
 					fmt.Printf("%s", err)
 					continue
 				}
-				fmt.Fprintf(w, logPayload.Log)
+				if *containerd {
+					// Due a containerd log format we need to add a \n for each line writed
+					fmt.Fprintln(w, logPayload.Log)
+				} else {
+					fmt.Fprintf(w, logPayload.Log)
+				}
 			}
 		}
 	})