refactor(userspace/engine): decoupling ruleset reading, parsing, and compilation steps

userspace/engine/CMakeLists.txt

@@ -19,6 +19,7 @@ set(FALCO_ENGINE_SOURCE_FILES
     formats.cpp
     filter_macro_resolver.cpp
     rule_loader.cpp
+    rule_reader.cpp
     stats_manager.cpp)
 
 add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES})

userspace/engine/falco_engine.cpp

@@ -25,6 +25,7 @@ limitations under the License.
 #include "falco_engine.h"
 #include "falco_utils.h"
 #include "falco_engine_version.h"
+#include "rule_reader.h"
 
 #include "formats.h"
 
@@ -51,6 +52,7 @@ falco_engine::falco_engine(bool seed_rng)
 
 falco_engine::~falco_engine()
 {
+	m_rules.clear();
 	m_rule_loader.clear();
 	m_rule_stats_manager.clear();
 }
@@ -148,23 +150,33 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 
 void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version)
 {
-	std::vector<std::string> warnings;
-	std::vector<std::string> errors;
-	m_rule_loader.configure(m_min_priority, m_replace_container_info, m_extra);
-	bool success = m_rule_loader.load(rules_content, this, warnings, errors);
+	rule_loader::configuration cfg(rules_content);
+	cfg.engine = this;
+	cfg.min_priority = m_min_priority;
+	cfg.output_extra = m_extra;
+	cfg.replace_output_container_info = m_replace_container_info;
+
 	std::ostringstream os;
-	if (!errors.empty())
+	rule_reader reader;
+	bool success = reader.load(cfg, m_rule_loader);
+	if (success)
+	{
+		clear_filters();
+		m_rules.clear();
+		success = m_rule_loader.compile(cfg, m_rules);
+	}
+	if (!cfg.errors.empty())
 	{
-		os << errors.size() << " errors:" << std::endl;
-		for(auto &err : errors)
+		os << cfg.errors.size() << " errors:" << std::endl;
+		for(auto &err : cfg.errors)
 		{
 			os << err << std::endl;
 		}
 	}
-	if (!warnings.empty())
+	if (!cfg.warnings.empty())
 	{
-		os << warnings.size() << " warnings:" << std::endl;
-		for(auto &warn : warnings)
+		os << cfg.warnings.size() << " warnings:" << std::endl;
+		for(auto &warn : cfg.warnings)
 		{
 			os << warn << std::endl;
 		}
@@ -315,7 +327,7 @@ unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t so
 		unique_ptr<struct rule_result> res(new rule_result());
 		// note: indexes are 0-based, whereas check_ids are not
 		auto rule_idx = ev->get_check_id() - 1;
-		auto rule = m_rule_loader.rules().at(rule_idx);
+		auto rule = m_rules.at(rule_idx);
 		if (!rule)
 		{
 			throw falco_exception("populate_rule_result error: unknown rule id "
@@ -328,7 +340,7 @@ unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t so
 		res->priority_num = rule->priority;
 		res->tags = rule->tags;
 		res->exception_fields = rule->exception_fields;
-		m_rule_stats_manager.on_event(m_rule_loader.rules(), rule_idx);
+		m_rule_stats_manager.on_event(m_rules, rule_idx);
 		return res;
 	}
 	catch(std::out_of_range const &exc)
@@ -374,15 +386,15 @@ void falco_engine::describe_rule(string *rule)
 	fprintf(stdout, rule_fmt, "----",  "-----------");
 	if (!rule)
 	{
-		for (auto &r : m_rule_loader.rules())
+		for (auto &r : m_rules)
 		{
 			auto str = falco::utils::wrap_text(r.description, 51, 110) + "\n";
 			fprintf(stdout, rule_fmt, r.name.c_str(), str.c_str());
 		}
 	}
 	else
 	{
-		auto r = m_rule_loader.rules().at(*rule);
+		auto r = m_rules.at(*rule);
 		auto str = falco::utils::wrap_text(r->description, 51, 110) + "\n";
 		fprintf(stdout, rule_fmt, r->name.c_str(), str.c_str());
 	}
@@ -392,7 +404,7 @@ void falco_engine::describe_rule(string *rule)
 void falco_engine::print_stats()
 {
 	string out;
-	m_rule_stats_manager.format(m_rule_loader.rules(), out);
+	m_rule_stats_manager.format(m_rules, out);
 	// todo(jasondellaluce): introduce a logging callback in Falco
 	fprintf(stdout, "%s", out.c_str());
 }
@@ -432,6 +444,11 @@ void falco_engine::clear_filters()
 	}
 }
 
+void falco_engine::clear_loader()
+{
+	m_rule_loader.clear();
+}
+
 void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)
 {
 	m_sampling_ratio = sampling_ratio;

userspace/engine/falco_engine.h

@@ -118,6 +118,15 @@ class falco_engine
 	// Clear all existing filters.
 	void clear_filters();
 
+	//
+	// Clear all the definitions of the internal rule loader (e.g. defined
+	// rules, macros, lists, engine/plugin version requirements). This is meant
+	// to be used to free-up memory at runtime when the definitions are not
+	// used anymore. Calling this between successive invocations of load_rules
+	// or load_rules_file can cause failures of features like appending.
+	//
+	void clear_loader();
+
 	//
 	// Set the sampling ratio, which can affect which events are
 	// matched against the set of rules.
@@ -248,6 +257,7 @@ class falco_engine
 	std::vector<ruleset_node> m_rulesets;
 
 	rule_loader m_rule_loader;
+	indexed_vector<falco_rule> m_rules;
 	stats_manager m_rule_stats_manager;
 
 	uint16_t m_next_ruleset_id;