bug non utf 8 byte sequences error with color output

CHANGELOG-Japanese.md

@@ -1,6 +1,6 @@
 # 変更点
 
-## v1.4.0 [2022/XX/XX]
+## v1.4.0 [2022/06/26]
 
 **新機能:**
 
@@ -9,14 +9,15 @@
 
 **改善:**
 
+- ルール内に`details`フィールドがないときに、`rules/config/default_details.txt`に設定されたデフォルトの出力を行えるようにした。 (#359) (@hitenkoku)
 - Clap Crateパッケージの更新 (#413) (@hitenkoku)
 - オプションの指定がないときに、`--help`と同じ画面出力を行うように変更した。(#387) (@hitenkoku)
-- ルール内に`details`フィールドがないときに、`rules/config/default_details.txt`に設定されたデフォルトの出力を行えるようにした。 (#359) (@hitenkoku)
 - `output` オプションで指定されファイルのサイズを出力するようにした。 (#595) (@hitenkoku)
 
 **バグ修正:**
 
-- XXX
+- カラー出力で長い出力があった場合にエラーが出て終了する問題を修正した。 (#603) (@hitenkoku)
+- `Excluded rules`の合計で`rules/tools/sigmac/testfiles`配下のテストルールも入っていたので、無視するようにした。 (#602) (@hitenkoku)
 
 ## v1.3.2 [2022/06/13]
 

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Changes
 
-## v1.4.0 [2022/XX/XX]
+## v1.4.0 [2022/06/26]
 
 **New Features:**
 
@@ -9,15 +9,15 @@
 
 **Enhancements:**
 
+- Added default details output based on `rules/config/default_details.txt` when no `details` field in a rule is specified. (i.e. Sigma rules) (#359) (@hitenkoku)
 - Updated clap crate package to version 3. (#413) (@hitnekoku)
 - Updated the default usage and help menu. (#387) (@hitenkoku)
-- Added default details output based on `rules/config/default_details.txt` when no `details` field in a rule is specified. (i.e. Sigma rules) (#359) (@hitenkoku)
 - Added saved file size output when `output` is specified. (#595) (@hitenkoku)
-- Ignored loading yml file in `rules/tools/sigmac/testfiles`. (#602) (@hitenkoku)
 
 **Bug Fixes:**
 
-- XXX
+- Fixed output error and program termination when long output is displayed with color. (#603) (@hitenkoku)
+- Ignore loading yml files in `rules/tools/sigmac/testfiles` to fix `Excluded rules` count. (#602) (@hitenkoku)
 
 ## v1.3.2 [2022/06/13]
 

src/afterfact.rs

@@ -283,18 +283,20 @@ fn emit_csv<W: std::io::Write>(
 
                 //ヘッダーのみを出力
                 if plus_header {
-                    write!(disp_wtr_buf, "{}", _get_serialized_disp_output(None)).ok();
-                    plus_header = false;
-                }
-                disp_wtr_buf
-                    .set_color(
-                        ColorSpec::new().set_fg(_get_output_color(&color_map, &detect_info.level)),
+                    write_color_buffer(
+                        &disp_wtr,
+                        get_writable_color(None),
+                        &_get_serialized_disp_output(None),
+                        true,
                     )
                     .ok();
-                write!(
-                    disp_wtr_buf,
-                    "{}",
-                    _get_serialized_disp_output(Some(dispformat))
+                    plus_header = false;
+                }
+                write_color_buffer(
+                    &disp_wtr,
+                    get_writable_color(_get_output_color(&color_map, &detect_info.level)),
+                    &_get_serialized_disp_output(Some(dispformat)),
+                    false,
                 )
                 .ok();
             } else {
@@ -354,7 +356,6 @@ fn emit_csv<W: std::io::Write>(
         }
     }
     if displayflag {
-        disp_wtr.print(&disp_wtr_buf)?;
         println!();
     } else {
         wtr.flush()?;
@@ -384,6 +385,7 @@ fn emit_csv<W: std::io::Write>(
         &disp_wtr,
         get_writable_color(Some(Color::Green)),
         "Results Summary:",
+        true,
     )
     .ok();
 
@@ -407,6 +409,7 @@ fn emit_csv<W: std::io::Write>(
         &disp_wtr,
         get_writable_color(None),
         &format!("Total events: {}", all_record_cnt),
+        true,
     )
     .ok();
     write_color_buffer(
@@ -416,6 +419,7 @@ fn emit_csv<W: std::io::Write>(
             "Data reduction: {} events ({:.2}%)",
             reducted_record_cnt, reducted_percent
         ),
+        true,
     )
     .ok();
     println!();
@@ -472,7 +476,7 @@ fn _get_serialized_disp_output(dispformat: Option<DisplayFormat>) -> String {
         if configs::CONFIG.read().unwrap().args.full_data {
             titles.push("RecordInformation");
         }
-        return format!("{}\n", titles.join("|"));
+        return titles.join("|");
     }
     let mut disp_serializer = csv::WriterBuilder::new()
         .double_quote(false)
@@ -522,8 +526,9 @@ fn _print_unique_results(
             "{} {}: {}",
             head_word,
             tail_word,
-            counts_by_level.iter().sum::<u128>()
+            counts_by_level.iter().sum::<u128>(),
         ),
+        true,
     )
     .ok();
 
@@ -539,6 +544,7 @@ fn _print_unique_results(
             &BufferWriter::stdout(ColorChoice::Always),
             _get_output_color(color_map, level_name),
             &output_raw_str,
+            true,
         )
         .ok();
     }
@@ -821,7 +827,7 @@ mod tests {
         let test_timestamp = Utc
             .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ")
             .unwrap();
-        let expect_header = "Timestamp|Computer|Channel|EventID|Level|RecordID|RuleTitle|Details\n";
+        let expect_header = "Timestamp|Computer|Channel|EventID|Level|RecordID|RuleTitle|Details";
         let expect_tz = test_timestamp.with_timezone(&Local);
 
         let expect_no_header = expect_tz
@@ -845,7 +851,7 @@ mod tests {
             + "|"
             + test_recinfo
             + "\n";
-        assert_eq!(_get_serialized_disp_output(None,), expect_header);
+        assert_eq!(_get_serialized_disp_output(None), expect_header);
         assert_eq!(
             _get_serialized_disp_output(Some(DisplayFormat {
                 timestamp: &format_time(&test_timestamp, false),

src/detections/detection.rs

@@ -394,6 +394,7 @@ impl Detection {
                 &BufferWriter::stdout(ColorChoice::Always),
                 Some(Color::Red),
                 &format!("Rule parsing errors: {}", err_rc),
+                true,
             )
             .ok();
         }
@@ -421,6 +422,7 @@ impl Detection {
                         rate,
                         deprecated_flag
                     ),
+                    true,
                 )
                 .ok();
             }
@@ -434,6 +436,7 @@ impl Detection {
                 &BufferWriter::stdout(ColorChoice::Always),
                 None,
                 &format!("{} rules: {}", key, value),
+                true,
             )
             .ok();
         });

src/detections/print.rs

@@ -327,6 +327,7 @@ impl AlertMessage {
             &BufferWriter::stderr(ColorChoice::Always),
             None,
             &format!("[ERROR] {}", contents),
+            true,
         )
     }
 
@@ -336,6 +337,7 @@ impl AlertMessage {
             &BufferWriter::stderr(ColorChoice::Always),
             None,
             &format!("[WARN] {}", contents),
+            true,
         )
     }
 }

src/detections/utils.rs

@@ -245,10 +245,15 @@ pub fn write_color_buffer(
     wtr: &BufferWriter,
     color: Option<Color>,
     output_str: &str,
+    newline_flag: bool,
 ) -> io::Result<()> {
     let mut buf = wtr.buffer();
     buf.set_color(ColorSpec::new().set_fg(color)).ok();
-    writeln!(buf, "{}", output_str).ok();
+    if newline_flag {
+        writeln!(buf, "{}", output_str).ok();
+    } else {
+        write!(buf, "{}", output_str).ok();
+    }
     wtr.print(&buf)
 }
 

src/main.rs

@@ -120,6 +120,7 @@ impl App {
                             &BufferWriter::stdout(ColorChoice::Always),
                             None,
                             "Rules updated successfully.",
+                            true,
                         )
                         .ok();
                     }
@@ -173,6 +174,7 @@ impl App {
                 &BufferWriter::stdout(ColorChoice::Always),
                 None,
                 "Generating Event ID Statistics",
+                true,
             )
             .ok();
             println!();
@@ -182,6 +184,7 @@ impl App {
                 &BufferWriter::stdout(ColorChoice::Always),
                 None,
                 "Generating Logons Summary",
+                true,
             )
             .ok();
             println!();
@@ -265,6 +268,7 @@ impl App {
                 &BufferWriter::stdout(ColorChoice::Always),
                 None,
                 &configs::CONFIG.read().unwrap().headless_help,
+                true,
             )
             .ok();
             return;
@@ -277,6 +281,7 @@ impl App {
             &BufferWriter::stdout(ColorChoice::Always),
             None,
             &format!("Elapsed Time: {}", &analysis_duration.hhmmssxxx()),
+            true,
         )
         .ok();
         println!();
@@ -329,17 +334,30 @@ impl App {
                     )
                     .ok();
                 });
-                write_color_buffer(&BufferWriter::stdout(ColorChoice::Always), None, &output).ok();
+                write_color_buffer(
+                    &BufferWriter::stdout(ColorChoice::Always),
+                    None,
+                    &output,
+                    true,
+                )
+                .ok();
             } else {
                 //標準出力の場合
                 let output = "The following pivot keywords were found:".to_string();
-                write_color_buffer(&BufferWriter::stdout(ColorChoice::Always), None, &output).ok();
+                write_color_buffer(
+                    &BufferWriter::stdout(ColorChoice::Always),
+                    None,
+                    &output,
+                    true,
+                )
+                .ok();
 
                 pivot_key_unions.iter().for_each(|(key, pivot_keyword)| {
                     write_color_buffer(
                         &BufferWriter::stdout(ColorChoice::Always),
                         None,
                         &create_output(String::default(), key, pivot_keyword),
+                        true,
                     )
                     .ok();
                 });
@@ -425,8 +443,13 @@ impl App {
     fn print_contributors(&self) {
         match fs::read_to_string("./contributors.txt") {
             Ok(contents) => {
-                write_color_buffer(&BufferWriter::stdout(ColorChoice::Always), None, &contents)
-                    .ok();
+                write_color_buffer(
+                    &BufferWriter::stdout(ColorChoice::Always),
+                    None,
+                    &contents,
+                    true,
+                )
+                .ok();
             }
             Err(err) => {
                 AlertMessage::alert(&format!("{}", err)).ok();
@@ -445,6 +468,7 @@ impl App {
             &BufferWriter::stdout(ColorChoice::Always),
             None,
             &format!("Analyzing event files: {:?}", evtx_files.len()),
+            true,
         )
         .ok();
 
@@ -671,6 +695,7 @@ impl App {
             &BufferWriter::stdout(ColorChoice::Always),
             output_color,
             &content,
+            true,
         )
         .ok();
     }
@@ -687,7 +712,13 @@ impl App {
             None => {}
             Some(path) => {
                 let content = fs::read_to_string(path).unwrap_or_default();
-                write_color_buffer(&BufferWriter::stdout(ColorChoice::Always), None, &content).ok();
+                write_color_buffer(
+                    &BufferWriter::stdout(ColorChoice::Always),
+                    None,
+                    &content,
+                    true,
+                )
+                .ok();
             }
         }
     }
@@ -704,6 +735,7 @@ impl App {
                 &BufferWriter::stdout(ColorChoice::Always),
                 None,
                 "Attempting to git clone the hayabusa-rules repository into the rules folder.",
+                true,
             )
             .ok();
             // execution git clone of hayabusa-rules repository when failed open hayabusa repository.
@@ -886,6 +918,7 @@ impl App {
                     "[Updated] {} (Modified: {} | Path: {})",
                     tmp[0], tmp[1], tmp[2]
                 ),
+                true,
             )
             .ok();
         }
@@ -900,6 +933,7 @@ impl App {
                 &BufferWriter::stdout(ColorChoice::Always),
                 None,
                 "You currently have the latest rules.",
+                true,
             )
             .ok();
             Ok("You currently have the latest rules.".to_string())

src/options/level_tuning.rs

@@ -62,6 +62,7 @@ impl LevelTuning {
                     &BufferWriter::stdout(ColorChoice::Always),
                     None,
                     &format!("path: {}", path),
+                    true,
                 )
                 .ok();
                 let mut content = match fs::read_to_string(&path) {
@@ -101,6 +102,7 @@ impl LevelTuning {
                         rule["level"].as_str().unwrap(),
                         new_level
                     ),
+                    true,
                 )
                 .ok();
             }