Support extract timestamp from patterned datetime string in LAL.

docs/en/changes/changes.md

@@ -35,6 +35,7 @@
 * BanyanDBStorageClient: Add `define(Property property, PropertyStore.Strategy strategy)` API.
 * Support GraalVM native-image (Experimental).
 * Correct the file format and fix typos in the filenames for monitoring Kafka's e2e tests.
+* Support extract timestamp from patterned datetime string in LAL.
 
 #### UI
 

docs/en/concepts-and-designs/lal.md

@@ -185,7 +185,26 @@ dropped) and is used to associate with traces / metrics.
 `timestamp` extracts the timestamp from the `parsed` result, and set it into the `LogData`, which will be persisted (if
 not dropped) and is used to associate with traces / metrics.
 
-The unit of `timestamp` is millisecond.
+The parameter of `timestamp` can be a millisecond:
+```groovy
+filter {
+    // ... parser
+
+    extractor {
+        timestamp parsed.time as String
+    }
+}
+```
+or a datetime string with specified pattern:
+```groovy
+filter {
+    // ... parser
+
+    extractor {
+        timestamp parsed.time, "yyyy-MM-dd HH:mm:ss"
+    }
+}
+```
 
 - `layer`
 

oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java

@@ -22,6 +22,8 @@
 import com.google.common.collect.ImmutableMap;
 import groovy.lang.Closure;
 import groovy.lang.DelegatesTo;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -57,6 +59,7 @@
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.module.ModuleStartException;
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -203,11 +206,29 @@ public void spanId(final String spanId) {
 
     @SuppressWarnings("unused")
     public void timestamp(final String timestamp) {
+        timestamp(timestamp, null);
+    }
+
+    @SuppressWarnings("unused")
+    public void timestamp(final String timestamp, final String formatPattern) {
         if (BINDING.get().shouldAbort()) {
             return;
         }
-        if (nonNull(timestamp) && StringUtils.isNumeric(timestamp)) {
-            BINDING.get().log().setTimestamp(Long.parseLong(timestamp));
+        if (StringUtil.isEmpty(timestamp)) {
+            return;
+        }
+
+        if (StringUtil.isEmpty(formatPattern)) {
+            if (StringUtils.isNumeric(timestamp)) {
+                BINDING.get().log().setTimestamp(Long.parseLong(timestamp));
+            }
+        } else {
+            SimpleDateFormat format = new SimpleDateFormat(formatPattern);
+            try {
+                BINDING.get().log().setTimestamp(format.parse(timestamp).getTime());
+            } catch (ParseException e) {
+                // ignore
+            }
         }
     }
 

oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java

@@ -176,6 +176,17 @@ public static Collection<Object[]> data() {
                                 "          }\n" +
                                 "        }\n" +
                                 "      }"
+                },
+                new String[] {
+                    "extractor-patterned-timestamp",
+                    "filter {\n" +
+                        "  extractor {\n" +
+                        "    service \"test\"\n" +
+                        "    instance \"test\"\n" +
+                        "    endpoint \"test\"\n" +
+                        "    timestamp \"2023-11-01 22:10:10\", \"yyyy-MM-dd HH:mm:ss\"\n" +
+                        "  }\n" +
+                        "}",
                 }
         );
     }

test/e2e-v2/cases/log/extract-timestamp/docker-compose.yml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+  h2db:
+    build:
+      context: .
+      dockerfile: ../../../script/dockerfile/Dockerfile.h2
+    networks:
+      - e2e
+    expose:
+      - 1521
+    healthcheck:
+      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/1521"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+  oap:
+    extends:
+      file: ../log-base-compose.yml
+      service: oap
+    environment:
+      SW_STORAGE: h2
+      SW_STORAGE_H2_URL: jdbc:h2:tcp://h2db:1521/skywalking-oap-db;DATABASE_TO_UPPER=FALSE
+    volumes:
+      - ./lal.yaml:/skywalking/config/lal/test.yaml
+      - ../log-mal.yaml:/skywalking/config/log-mal-rules/test.yaml
+    depends_on:
+      h2db:
+        condition: service_healthy
+    ports:
+      - 12800
+    networks:
+      - e2e
+
+  provider:
+    extends:
+      file: ../../../script/docker-compose/base-compose.yml
+      service: provider
+    ports:
+      - 9090
+    networks:
+      - e2e
+    depends_on:
+      oap:
+        condition: service_healthy
+
+networks:
+  e2e:

test/e2e-v2/cases/log/extract-timestamp/e2e.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+setup:
+  env: compose
+  file: docker-compose.yml
+  timeout: 20m
+  init-system-environment: ../../../script/env
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install yq
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq
+    - name: install swctl
+      command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl
+
+trigger:
+  action: http
+  interval: 3s
+  times: 10
+  url: http://${provider_host}:${provider_9090}/logs/trigger
+  method: GET
+
+verify:
+  # verify with retry strategy
+  retry:
+    # max retry count
+    count: 20
+    # the interval between two retries, in millisecond.
+    interval: 10s
+  cases:
+    - includes:
+        - ../log-cases.yaml

test/e2e-v2/cases/log/extract-timestamp/lal.yaml

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rules:
+  - name: example
+    layer: GENERAL
+    dsl: |
+      filter {
+        text {
+          abortOnFailure false // for test purpose, we want to persist all logs
+          regexp $/(?s)(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}) \[TID:(?<tid>.+?)] \[(?<thread>.+?)] (?<level>\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$
+        }
+        extractor {
+          timestamp parsed.timestamp as String, "yyyy-MM-dd HH:mm:ss.SSS"
+          metrics {
+            timestamp log.timestamp as Long
+            labels level: parsed.level, service: log.service, instance: log.serviceInstance
+            name "log_count"
+            value 1
+          }
+        }
+        sink {
+        }
+      }