feat:Added the serverlog periodic reclamation function

conf/pika.conf

@@ -64,6 +64,11 @@ sync-binlog-thread-num : 1
 # is used for replication.
 log-path : ./log/
 
+# log retention time of serverlogs(pika.{hostname}.{username}.log.{loglevel}.YYYYMMDD-HHMMSS) files that stored within log-path.
+# Any serverlogs files that exceed this time will be cleaned up.
+# The unit of serverlogs is in [days] and the default value is 7(days).
+log-retention-time : 7
+
 # Directory to store the data of Pika.
 db-path : ./db/
 

include/pika_conf.h

@@ -77,6 +77,10 @@ class PikaConf : public pstd::BaseConf {
     std::shared_lock l(rwlock_);
     return log_path_;
   }
+  int log_retention_time() {
+    std::shared_lock l(rwlock_);
+    return log_retention_time_;
+  }
   std::string log_level() {
     std::shared_lock l(rwlock_);
     return log_level_;
@@ -904,6 +908,7 @@ class PikaConf : public pstd::BaseConf {
   int db_sync_speed_ = 0;
   std::string slaveof_;
   std::string log_path_;
+  int log_retention_time_;
   std::string log_level_;
   std::string db_path_;
   int db_instance_num_ = 0;

include/pika_server.h

@@ -513,7 +513,8 @@ class PikaServer : public pstd::noncopyable {
    */
   void DoTimingTask();
   void AutoCompactRange();
-  void AutoPurge();
+  void AutoBinlogPurge();
+  void AutoServerlogPurge();
   void AutoDeleteExpiredDump();
   void AutoUpdateNetworkMetric();
   void PrintThreadPoolQueueStatus();

src/pika_server.cc

@@ -1085,8 +1085,10 @@ int PikaServer::ClientPubSubChannelPatternSize(const std::shared_ptr<NetConn>& c
 void PikaServer::DoTimingTask() {
   // Maybe schedule compactrange
   AutoCompactRange();
-  // Purge log
-  AutoPurge();
+  // Purge serverlog
+  AutoServerlogPurge();
+  // Purge binlog
+  AutoBinlogPurge();
   // Delete expired dump
   AutoDeleteExpiredDump();
   // Cheek Rsync Status
@@ -1206,7 +1208,82 @@ void PikaServer::AutoCompactRange() {
   }
 }
 
-void PikaServer::AutoPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); }
+void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); }
+
+void PikaServer::AutoServerlogPurge(){
+std::string log_path = g_pika_conf->log_path();
+  int retention_time = g_pika_conf->log_retention_time();
+  if (retention_time <= 0) {
+    return;
+  }
+  std::vector<std::string> log_files;
+
+  if (!pstd::FileExists(log_path)) {
+    return;
+  }
+
+  if (pstd::GetChildren(log_path, log_files) != 0) {
+    return;
+  }
+
+  time_t t = time(nullptr);
+  struct tm* now = localtime(&t);
+  int now_year = now->tm_year + 1900;
+  int now_month = now->tm_mon + 1;
+  int now_day = now->tm_mday;
+
+  //logformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid]
+  for (const auto& file : log_files) {
+    if (file.substr(0, 5) != "pika.") {
+      continue;
+    }
+
+    size_t log_pos = file.find(".log.");
+    if (log_pos == std::string::npos) {
+      continue;
+    }
+
+    // Start at the end of ".log." to find the date and time
+    size_t date_pos = file.find('.', log_pos + 5);
+    if (date_pos == std::string::npos || date_pos + 16 > file.length()) {
+      continue;
+    }
+
+    std::string date = file.substr(date_pos + 1, 8);
+    std::string time = file.substr(date_pos + 10, 6);
+
+    int log_year = std::atoi(date.substr(0, 4).c_str());
+    int log_month = std::atoi(date.substr(4, 2).c_str());
+    int log_day = std::atoi(date.substr(6, 2).c_str());
+
+    struct tm log_time;
+    struct tm now_time;
+
+    log_time.tm_year = log_year - 1900;
+    log_time.tm_mon = log_month - 1;
+    log_time.tm_mday = log_day;
+    log_time.tm_hour = 0;
+    log_time.tm_min = 0;
+    log_time.tm_sec = 0;
+
+    now_time.tm_year = now_year - 1900;
+    now_time.tm_mon = now_month - 1;
+    now_time.tm_mday = now_day;
+    now_time.tm_hour = 0;
+    now_time.tm_min = 0;
+    now_time.tm_sec = 0;
+
+    int64_t log_timestamp = mktime(&log_time);
+    int64_t now_timestamp = mktime(&now_time);
+    int64_t interval_days = (now_timestamp - log_timestamp) / 86400;
+
+    if (interval_days > retention_time) {
+      std::string log_file = log_path + "/" + file;
+      LOG(INFO) << "Deleting out of date log file: " << log_file;
+      pstd::DeleteFile(log_file);
+    }
+  }
+}
 
 void PikaServer::AutoDeleteExpiredDump() {
   std::string db_sync_prefix = g_pika_conf->bgsave_prefix();

tools/kubeblocks_helm/pika-cluster/templates/cluster.yaml

@@ -6,7 +6,7 @@ metadata:
   labels: {{ include "pika-cluster.labels" . | nindent 4 }}
 spec:
   clusterDefinitionRef: pika  # ref clusterDefinition.name
-  clusterVersionRef: pika-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterVersion.name
+  topology: replication-codis
   terminationPolicy: {{ .Values.terminationPolicy }}
   affinity:
     {{- with .Values.topologyKeys }}
@@ -16,13 +16,14 @@ spec:
   tolerations: {{ . | toYaml | nindent 4 }}
   {{- end }}
   ## define pika group with shardingSpecs API which is supported in KubeBlocks v0.8.2
-  {{- if not .Values.useLegacyCompDef }}
   shardingSpecs:
   - name: group
     shards: {{ .Values.groupCount }}
     template:
       name: pika
       componentDef: pika-group
+      enabledLogs: {{ $.Values.enabledLogs | toJson | indent 4 }}
+      serviceAccountName: {{ include "pika-cluster.serviceAccountName" $ }}
       replicas: {{ add (int $.Values.slaveCount) 1 | default 2 }}
       {{- with  $.Values.resources.pikaGroup }}
       resources:
@@ -46,12 +47,9 @@ spec:
                 storage: {{ .size }}
         {{- end }}
       {{- end }}
-  {{- end }}
   componentSpecs:
-    {{- if .Values.useLegacyCompDef }}
-    {{- range $i := until (int .Values.groupCount) }}
-    - name: pika-group-{{ add ($i) 1 }} # user-defined
-      componentDefRef: pika-group # ref clusterDefinition.componentDefs[x].name
+    - name: pika-group # user-defined
+      componentDef: pika-group # ref clusterDefinition.componentDefs[x].name
       monitor: {{ $.Values.monitor.enabled | default false }}
       enabledLogs: {{ $.Values.enabledLogs | toJson | indent 4 }}
       replicas: {{ add (int $.Values.slaveCount) 1 | default 2 }}
@@ -80,15 +78,9 @@ spec:
                 storage: {{ .size }}
         {{- end }}
       {{- end }}
-      {{- end }}
-      {{- end }}
     - name: etcd # user-defined
-      {{- if .Values.useLegacyCompDef }}
-      componentDefRef: etcd # ref clusterDefinition.componentDefs[x].name
-      {{- else }}
       componentDef: pika-etcd # ref componentDefinition name
-      {{- end }}
-      monitor: {{ .Values.monitor.enabled | default false }}
+      #monitor: {{ .Values.monitor.enabled | default false }}
       replicas: {{ .Values.etcdReplicaCount| default 3 }}
       {{- with  .Values.resources.etcd }}
       resources:
@@ -116,29 +108,8 @@ spec:
                 storage: {{ .size }}
         {{- end }}
       {{- end }}
-    - name: pika-exporter
-      {{- if .Values.useLegacyCompDef }}
-      componentDefRef: pika-exporter # ref clusterDefinition.componentDefs[x].name
-      {{- else }}
-      componentDef: pika-exporter # ref componentDefinition name
-      {{- end }}
-      monitor: {{ .Values.monitor.enabled | default false }}
-      replicas: 1
-      {{- with  .Values.resources.pikaExporter }}
-      resources:
-        limits:
-          cpu: {{ .limits.cpu | quote }}
-          memory: {{ .limits.memory | quote }}
-        requests:
-          cpu: {{ .requests.cpu | quote }}
-          memory: {{ .requests.memory | quote }}
-      {{- end }}
     - name: codis-proxy
-      {{- if .Values.useLegacyCompDef }}
-      componentDefRef: codis-proxy # ref clusterDefinition.componentDefs[x].name
-      {{- else }}
       componentDef: pika-codis-proxy # ref componentDefinition name
-      {{- end }}
       replicas: {{ .Values.codisProxyReplicaCount | default 2 }}
       {{- with  .Values.resources.codisProxy }}
       resources:
@@ -150,11 +121,7 @@ spec:
           memory: {{ .requests.memory | quote }}
       {{- end }}
     - name: codis-fe
-      {{- if .Values.useLegacyCompDef }}
-      componentDefRef: codis-fe # ref clusterDefinition.componentDefs[x].name
-      {{- else }}
       componentDef: pika-codis-fe # ref componentDefinition name
-      {{- end }}
       replicas: {{ .Values.codisFeReplicaCount | default 1 }}
       {{- with  .Values.resources.codisFe }}
       resources:
@@ -166,11 +133,7 @@ spec:
           memory: {{ .requests.memory | quote }}
       {{- end }}
     - name: codis-dashboard
-      {{- if .Values.useLegacyCompDef }}
-      componentDefRef: codis-dashboard # ref clusterDefinition.componentDefs[x].name
-      {{- else }}
       componentDef: pika-codis-dashboard # ref componentDefinition name
-      {{- end }}
       replicas: 1
     {{- with  .Values.resources.codisFe }}
       resources:

tools/kubeblocks_helm/pika/.helmignore

@@ -10,7 +10,7 @@
 .hg/
 .hgignore
 .svn/
-# Common backup files
+# Common  files
 *.swp
 *.bak
 *.tmp

tools/kubeblocks_helm/pika/script/admin.sh

@@ -9,7 +9,7 @@ set_instance_role() {
 
 # set group id
 set_group_id() {
-  GROUP_ID=${KB_CLUSTER_COMP_NAME##*-}
+  GROUP_ID=${KB_POD_NAME##*-}
   echo "GROUP_ID: "${GROUP_ID}
 }