Address review comments from #5596

CHANGELOG.rst

@@ -152,7 +152,7 @@ Added
 
   Contributed by @Kami.
 
-* Added garbage collection for rule_enforcement and trace models #5596
+* Added garbage collection for rule_enforcement and trace models #5596/5602
   Contributed by Amanda McGuinness (@amanda11 intive)
 
 

conf/st2.conf.sample

@@ -167,23 +167,23 @@ dump_dir = /opt/stackstorm/exports/
 logging = /etc/st2/logging.exporter.conf
 
 [garbagecollector]
-# Action execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted.
+# Action execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted. Defaults to 7.
 action_executions_output_ttl = 7
-# Action executions and related objects (live actions, action output objects) older than this value (days) will be automatically deleted.
+# Action executions and related objects (live actions, action output objects) older than this value (days) will be automatically deleted. Defaults to None (disabled).
 action_executions_ttl = None
 # How often to check database for old data and perform garbage collection.
 collection_interval = 600
 # Location of the logging configuration file.
 logging = /etc/st2/logging.garbagecollector.conf
 # Set to True to perform garbage collection on Inquiries (based on the TTL value per Inquiry)
 purge_inquiries = False
-# Rule enforcements older than this value (days) will be automatically deleted.
-rule_enforcement_ttl = None
+# Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).
+rule_enforcements_ttl = None
 # How long to wait / sleep (in seconds) between collection of different object types.
 sleep_delay = 2
-# Trace objects older than this value (days) will be automatically deleted.
-trace_ttl = None
-# Trigger instances older than this value (days) will be automatically deleted.
+# Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).
+traces_ttl = None
+# Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).
 trigger_instances_ttl = None
 
 [keyvalue]

st2common/bin/st2-purge-rule-enforcement → st2common/bin/st2-purge-rule-enforcements

@@ -16,7 +16,7 @@
 
 import sys
 
-from st2common.cmd.purge_rule_enforcement import main
+from st2common.cmd.purge_rule_enforcements import main
 
 if __name__ == "__main__":
     sys.exit(main())

st2common/bin/st2-purge-trace → st2common/bin/st2-purge-traces

@@ -16,7 +16,7 @@
 
 import sys
 
-from st2common.cmd.purge_trace import main
+from st2common.cmd.purge_traces import main
 
 if __name__ == "__main__":
     sys.exit(main())

st2common/setup.py

@@ -53,8 +53,8 @@
         "bin/st2-register-content",
         "bin/st2-purge-executions",
         "bin/st2-purge-trigger-instances",
-        "bin/st2-purge-trace",
-        "bin/st2-purge-rule-enforcement",
+        "bin/st2-purge-traces",
+        "bin/st2-purge-rule-enforcements",
         "bin/st2-run-pack-tests",
         "bin/st2ctl",
         "bin/st2-generate-symmetric-crypto-key",

st2common/st2common/cmd/purge_rule_enforcement.py → st2common/st2common/cmd/purge_rule_enforcements.py

@@ -35,7 +35,7 @@
 from st2common.script_setup import teardown as common_teardown
 from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
 from st2common.constants.exit_codes import FAILURE_EXIT_CODE
-from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement
+from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements
 
 __all__ = ["main"]
 
@@ -71,7 +71,7 @@ def main():
 
     # Purge models.
     try:
-        purge_rule_enforcement(logger=LOG, timestamp=timestamp)
+        purge_rule_enforcements(logger=LOG, timestamp=timestamp)
     except Exception as e:
         LOG.exception(six.text_type(e))
         return FAILURE_EXIT_CODE

st2common/st2common/cmd/purge_trace.py → st2common/st2common/cmd/purge_traces.py

@@ -35,7 +35,7 @@
 from st2common.script_setup import teardown as common_teardown
 from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
 from st2common.constants.exit_codes import FAILURE_EXIT_CODE
-from st2common.garbage_collection.trace import purge_trace
+from st2common.garbage_collection.trace import purge_traces
 
 __all__ = ["main"]
 
@@ -71,7 +71,7 @@ def main():
 
     # Purge models.
     try:
-        purge_trace(logger=LOG, timestamp=timestamp)
+        purge_traces(logger=LOG, timestamp=timestamp)
     except Exception as e:
         LOG.exception(six.text_type(e))
         return FAILURE_EXIT_CODE

st2common/st2common/garbage_collection/rule_enforcement.py

@@ -24,10 +24,10 @@
 from st2common.persistence.rule_enforcement import RuleEnforcement
 from st2common.util import isotime
 
-__all__ = ["purge_rule_enforcement"]
+__all__ = ["purge_rule_enforcements"]
 
 
-def purge_rule_enforcement(logger, timestamp):
+def purge_rule_enforcements(logger, timestamp):
     """
     :param timestamp: Rule enforcement instances older than this timestamp will be deleted.
     :type timestamp: ``datetime.datetime

st2common/st2common/garbage_collection/trace.py

@@ -24,10 +24,10 @@
 from st2common.persistence.trace import Trace
 from st2common.util import isotime
 
-__all__ = ["purge_trace"]
+__all__ = ["purge_traces"]
 
 
-def purge_trace(logger, timestamp):
+def purge_traces(logger, timestamp):
     """
     :param timestamp: Trace instances older than this timestamp will be deleted.
     :type timestamp: ``datetime.datetime

st2common/tests/unit/test_purge_rule_enforcement.py

@@ -17,7 +17,7 @@
 import bson
 
 from st2common import log as logging
-from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement
+from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements
 from st2common.models.db.rule_enforcement import RuleEnforcementDB
 from st2common.persistence.rule_enforcement import RuleEnforcement
 from st2common.util import date as date_utils
@@ -46,7 +46,7 @@ def test_no_timestamp_doesnt_delete(self):
         self.assertRaisesRegexp(
             ValueError,
             expected_msg,
-            purge_rule_enforcement,
+            purge_rule_enforcements,
             logger=LOG,
             timestamp=None,
         )
@@ -63,7 +63,7 @@ def test_purge(self):
         )
 
         self.assertEqual(len(RuleEnforcement.get_all()), 2)
-        purge_rule_enforcement(logger=LOG, timestamp=now - timedelta(days=10))
+        purge_rule_enforcements(logger=LOG, timestamp=now - timedelta(days=10))
         self.assertEqual(len(RuleEnforcement.get_all()), 1)
 
     @staticmethod

st2common/tests/unit/test_purge_trace.py

@@ -17,7 +17,7 @@
 import bson
 
 from st2common import log as logging
-from st2common.garbage_collection.trace import purge_trace
+from st2common.garbage_collection.trace import purge_traces
 from st2common.models.db.trace import TraceDB, TraceComponentDB
 from st2common.persistence.trace import Trace
 from st2common.util import date as date_utils
@@ -50,7 +50,7 @@ def test_no_timestamp_doesnt_delete(self):
         self.assertRaisesRegexp(
             ValueError,
             expected_msg,
-            purge_trace,
+            purge_traces,
             logger=LOG,
             timestamp=None,
         )
@@ -75,7 +75,7 @@ def test_purge(self):
         )
 
         self.assertEqual(len(Trace.get_all()), 2)
-        purge_trace(logger=LOG, timestamp=now - timedelta(days=10))
+        purge_traces(logger=LOG, timestamp=now - timedelta(days=10))
         self.assertEqual(len(Trace.get_all()), 1)
 
     @staticmethod

st2reactor/st2reactor/garbage_collector/base.py

@@ -41,8 +41,8 @@
 from st2common.garbage_collection.executions import purge_orphaned_workflow_executions
 from st2common.garbage_collection.inquiries import purge_inquiries
 from st2common.garbage_collection.trigger_instances import purge_trigger_instances
-from st2common.garbage_collection.trace import purge_trace
-from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement
+from st2common.garbage_collection.trace import purge_traces
+from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements
 
 __all__ = ["GarbageCollectorService"]
 
@@ -71,8 +71,8 @@ def __init__(
             cfg.CONF.garbagecollector.action_executions_output_ttl
         )
         self._trigger_instances_ttl = cfg.CONF.garbagecollector.trigger_instances_ttl
-        self._trace_ttl = cfg.CONF.garbagecollector.trace_ttl
-        self._rule_enforcement_ttl = cfg.CONF.garbagecollector.rule_enforcement_ttl
+        self._traces_ttl = cfg.CONF.garbagecollector.traces_ttl
+        self._rule_enforcements_ttl = cfg.CONF.garbagecollector.rule_enforcements_ttl
         self._purge_inquiries = cfg.CONF.garbagecollector.purge_inquiries
         self._workflow_execution_max_idle = cfg.CONF.workflow_engine.gc_max_idle_sec
 
@@ -157,14 +157,17 @@ def _validate_ttl_values(self):
                 )
                 % (MINIMUM_TTL_DAYS_EXECUTION_OUTPUT)
             )
-        if self._trace_ttl and self._trace_ttl < MINIMUM_TTL_DAYS:
+        if self._traces_ttl and self._traces_ttl < MINIMUM_TTL_DAYS:
             raise ValueError(
-                "Minimum possible TTL for trace_ttl in days is %s" % (MINIMUM_TTL_DAYS)
+                "Minimum possible TTL for traces_ttl in days is %s" % (MINIMUM_TTL_DAYS)
             )
 
-        if self._rule_enforcement_ttl and self._rule_enforcement_ttl < MINIMUM_TTL_DAYS:
+        if (
+            self._rule_enforcements_ttl
+            and self._rule_enforcements_ttl < MINIMUM_TTL_DAYS
+        ):
             raise ValueError(
-                "Minimum possible TTL for rule_enforcement_ttl in days is %s"
+                "Minimum possible TTL for rule_enforcements_ttl in days is %s"
                 % (MINIMUM_TTL_DAYS)
             )
 
@@ -214,21 +217,21 @@ def _perform_garbage_collection(self):
 
         obj_type = "trace"
 
-        if self._trace_ttl and self._trace_ttl >= MINIMUM_TTL_DAYS:
+        if self._traces_ttl and self._traces_ttl >= MINIMUM_TTL_DAYS:
             LOG.info(proc_message, obj_type)
-            self._purge_trace()
+            self._purge_traces()
             concurrency.sleep(self._sleep_delay)
         else:
             LOG.debug(skip_message, obj_type)
 
         obj_type = "rule enforcement"
 
         if (
-            self._rule_enforcement_ttl
-            and self._rule_enforcement_ttl >= MINIMUM_TTL_DAYS
+            self._rule_enforcements_ttl
+            and self._rule_enforcements_ttl >= MINIMUM_TTL_DAYS
         ):
             LOG.info(proc_message, obj_type)
-            self._purge_rule_enforcement()
+            self._purge_rule_enforcements()
             concurrency.sleep(self._sleep_delay)
         else:
             LOG.debug(skip_message, obj_type)
@@ -342,12 +345,12 @@ def _purge_trigger_instances(self):
 
         return True
 
-    def _purge_trace(self):
+    def _purge_traces(self):
         """
         Purge trace objects which match the criteria defined in the config.
         """
         utc_now = get_datetime_utc_now()
-        timestamp = utc_now - datetime.timedelta(days=self._trace_ttl)
+        timestamp = utc_now - datetime.timedelta(days=self._traces_ttl)
 
         # Another sanity check to make sure we don't delete new objects
         if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
@@ -365,18 +368,18 @@ def _purge_trace(self):
             )
 
         try:
-            purge_trace(logger=LOG, timestamp=timestamp)
+            purge_traces(logger=LOG, timestamp=timestamp)
         except Exception as e:
             LOG.exception("Failed to delete trace: %s" % (six.text_type(e)))
 
         return True
 
-    def _purge_rule_enforcement(self):
+    def _purge_rule_enforcements(self):
         """
         Purge rule enforcements which match the criteria defined in the config.
         """
         utc_now = get_datetime_utc_now()
-        timestamp = utc_now - datetime.timedelta(days=self._rule_enforcement_ttl)
+        timestamp = utc_now - datetime.timedelta(days=self._rule_enforcements_ttl)
 
         # Another sanity check to make sure we don't delete new objects
         if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
@@ -394,7 +397,7 @@ def _purge_rule_enforcement(self):
             )
 
         try:
-            purge_rule_enforcement(logger=LOG, timestamp=timestamp)
+            purge_rule_enforcements(logger=LOG, timestamp=timestamp)
         except Exception as e:
             LOG.exception("Failed to delete rule enforcements: %s" % (six.text_type(e)))
 

st2reactor/st2reactor/garbage_collector/config.py

@@ -83,28 +83,28 @@ def _register_garbage_collector_opts(ignore_errors=False):
             "action_executions_ttl",
             default=None,
             help="Action executions and related objects (live actions, action output "
-            "objects) older than this value (days) will be automatically deleted.",
+            "objects) older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
         cfg.IntOpt(
             "action_executions_output_ttl",
             default=7,
             help="Action execution output objects (ones generated by action output "
-            "streaming) older than this value (days) will be automatically deleted.",
+            "streaming) older than this value (days) will be automatically deleted. Defaults to 7.",
         ),
         cfg.IntOpt(
             "trigger_instances_ttl",
             default=None,
-            help="Trigger instances older than this value (days) will be automatically deleted.",
+            help="Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
         cfg.IntOpt(
-            "rule_enforcement_ttl",
+            "rule_enforcements_ttl",
             default=None,
-            help="Rule enforcements older than this value (days) will be automatically deleted.",
+            help="Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
         cfg.IntOpt(
-            "trace_ttl",
+            "traces_ttl",
             default=None,
-            help="Trace objects older than this value (days) will be automatically deleted.",
+            help="Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
     ]
 

st2tests/st2tests/config.py

@@ -471,28 +471,28 @@ def _register_garbage_collector_opts():
             "action_executions_ttl",
             default=None,
             help="Action executions and related objects (live actions, action output "
-            "objects) older than this value (days) will be automatically deleted.",
+            "objects) older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
         cfg.IntOpt(
             "action_executions_output_ttl",
             default=7,
             help="Action execution output objects (ones generated by action output "
-            "streaming) older than this value (days) will be automatically deleted.",
+            "streaming) older than this value (days) will be automatically deleted. Defaults to 7.",
         ),
         cfg.IntOpt(
             "trigger_instances_ttl",
             default=None,
-            help="Trigger instances older than this value (days) will be automatically deleted.",
+            help="Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
         cfg.IntOpt(
-            "rule_enforcement_ttl",
+            "rule_enforcements_ttl",
             default=None,
-            help="Rule enforcements older than this value (days) will be automatically deleted.",
+            help="Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
         cfg.IntOpt(
-            "trace_ttl",
+            "traces_ttl",
             default=None,
-            help="Trace objects older than this value (days) will be automatically deleted.",
+            help="Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
     ]