From ae60d40f9f25d64b96cf60563ed7951bed163212 Mon Sep 17 00:00:00 2001 From: JIN Jie Date: Tue, 24 Sep 2024 15:18:12 +0800 Subject: [PATCH] chore: Improve rebuild speed by eliminating unnecessary rerun of build.rs - Watch libtrace changes by separate files instead of folders - Unwatch files generated from protobuf sources --- agent/Cargo.lock | 1 + agent/Cargo.toml | 2 + agent/build.rs | 104 +++++++++++++----- .../flow_generator/protocol_logs/mq/pulsar.rs | 1 + .../flow_generator/protocol_logs/rpc/brpc.rs | 1 + .../platform/platform_synchronizer/linux.rs | 5 +- agent/src/plugin/wasm/mod.rs | 1 + 7 files changed, 89 insertions(+), 26 deletions(-) diff --git a/agent/Cargo.lock b/agent/Cargo.lock index 68b31559627..c1e3e854332 100644 --- a/agent/Cargo.lock +++ b/agent/Cargo.lock @@ -1061,6 +1061,7 @@ dependencies = [ "tonic-build", "trace-utils", "uluru", + "walkdir", "wasmtime", "wasmtime-wasi", "winapi", diff --git a/agent/Cargo.toml b/agent/Cargo.toml index 02abafde39d..42962ff53f9 100644 --- a/agent/Cargo.toml +++ b/agent/Cargo.toml @@ -142,9 +142,11 @@ tempfile = "3.2.0" uluru = "3.0.0" [build-dependencies] +anyhow = "1.0" chrono = "0.4" dunce = "1" tonic-build = "0.8.0" +walkdir = "2" [features] enterprise = ["extended_profile", "enterprise-utils"] diff --git a/agent/build.rs b/agent/build.rs index dbb1d9286d8..b21b81e0d47 100644 --- a/agent/build.rs +++ b/agent/build.rs @@ -14,11 +14,18 @@ * limitations under the License. */ -use std::{env, error::Error, path::PathBuf, process::Command, str}; +use std::{ + env, + path::{Path, PathBuf}, + process::Command, + str, +}; +use anyhow::Result; use chrono::prelude::*; +use walkdir::WalkDir; -fn get_branch() -> Result> { +fn get_branch() -> Result { if let Ok(branch) = env::var("GITHUB_REF_NAME") { return Ok(branch); } @@ -60,7 +67,7 @@ fn get_branch() -> Result> { struct EnvCommand(&'static str, Vec<&'static str>); -fn set_build_info() -> Result<(), Box> { +fn set_build_info() -> Result<()> { println!("cargo:rustc-env=AGENT_NAME=deepflow-agent-ce"); println!("cargo:rustc-env=BRANCH={}", get_branch()?); println!( @@ -79,7 +86,61 @@ fn set_build_info() -> Result<(), Box> { Ok(()) } -fn set_build_libtrace() -> Result<(), Box> { +// libtrace scatters generated files in different folders, making it difficult to watch a single folder for changes +// +// rerun build script when one of the following file changes +// - C source files, except for +// - generated bpf bytecode files (socket_trace_*.c / perf_profiler_*.c) +// - java agent so files and jattach bin +// - Header files +// - `src/ebpf/mod.rs` (to exlude rust sources in `samples` folder) +// - Makefiles +fn set_libtrace_rerun_files() -> Result<()> { + fn watched(path: &Path) -> bool { + if let Some(ext) = path.extension().and_then(|s| s.to_str()) { + match ext { + "c" => { + if let Some(name) = path.file_name().and_then(|s| s.to_str()) { + if name.starts_with("socket_trace_") || name.starts_with("perf_profiler_") { + return false; + } + if name.starts_with("java_agent_so_") { + return false; + } + if name == "deepflow_jattach_bin.c" { + return false; + } + return true; + } + } + "h" => return true, + _ => (), + } + } + if path == Path::new("src/ebpf/mods.rs") { + return true; + } + if let Some(name) = path.file_name() { + if name == "Makefile" { + return true; + } + } + false + } + let base_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + for entry in WalkDir::new(base_dir.join("src/ebpf")) { + let entry = entry?; + let relative_path = entry.path().strip_prefix(&base_dir)?; + if !watched(relative_path) { + continue; + } + println!("cargo:rerun-if-changed={}", relative_path.display()); + } + Ok(()) +} + +fn set_build_libtrace() -> Result<()> { + set_libtrace_rerun_files()?; let output = match env::var("CARGO_CFG_TARGET_ENV")?.as_str() { "gnu" => Command::new("sh").arg("-c") .arg("cd src/ebpf && make clean && make --no-print-directory && make tools --no-print-directory") @@ -104,7 +165,7 @@ fn set_build_libtrace() -> Result<(), Box> { Ok(()) } -fn set_linkage() -> Result<(), Box> { +fn set_linkage() -> Result<()> { let target_env = env::var("CARGO_CFG_TARGET_ENV")?; if target_env.as_str() == "musl" { #[cfg(target_arch = "x86_64")] @@ -155,24 +216,28 @@ fn set_linkage() -> Result<(), Box> { Ok(()) } -fn compile_wasm_plugin_proto() -> Result<(), Box> { +fn compile_wasm_plugin_proto() -> Result<()> { tonic_build::configure() .build_server(false) + .emit_rerun_if_changed(false) .out_dir("src/plugin/wasm") - .compile(&["./src/plugin/WasmPluginApi.proto"], &["./src/plugin"])?; + .compile(&["src/plugin/WasmPluginApi.proto"], &["src/plugin"])?; + println!("cargo:rerun-if-changed=src/plugin/WasmPluginApi.proto"); Ok(()) } -fn make_pulsar_proto() -> Result<(), Box> { +fn make_pulsar_proto() -> Result<()> { tonic_build::configure() .field_attribute(".", "#[serde(skip_serializing_if = \"Option::is_none\")]") .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]") .build_server(false) + .emit_rerun_if_changed(false) .out_dir("src/flow_generator/protocol_logs/mq") .compile( &["src/flow_generator/protocol_logs/mq/PulsarApi.proto"], &["src/flow_generator/protocol_logs/mq"], )?; + println!("cargo:rerun-if-changed=src/flow_generator/protocol_logs/mq/PulsarApi.proto"); // remove `#[serde(skip_serializing_if = "Option::is_none")]` for non-optional fields let filename = "src/flow_generator/protocol_logs/mq/pulsar.proto.rs"; @@ -187,37 +252,26 @@ fn make_pulsar_proto() -> Result<(), Box> { new_lines.push(a[1]); } std::fs::write(filename, new_lines.join("\n"))?; - Command::new("cargo") - .args([ - "fmt", - "--", - "src/flow_generator/protocol_logs/mq/pulsar.proto.rs", - ]) - .spawn()?; Ok(()) } -fn make_brpc_proto() -> Result<(), Box> { +fn make_brpc_proto() -> Result<()> { tonic_build::configure() .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]") .build_server(false) + .emit_rerun_if_changed(false) .out_dir("src/flow_generator/protocol_logs/rpc/brpc") .compile( &["src/flow_generator/protocol_logs/rpc/brpc/baidu_rpc_meta.proto"], &["src/flow_generator/protocol_logs/rpc"], )?; - - Command::new("cargo") - .args([ - "fmt", - "--", - "src/flow_generator/protocol_logs/rpc/brpc.proto.rs", - ]) - .spawn()?; + println!( + "cargo:rerun-if-changed=src/flow_generator/protocol_logs/rpc/brpc/baidu_rpc_meta.proto" + ); Ok(()) } -fn main() -> Result<(), Box> { +fn main() -> Result<()> { set_build_info()?; compile_wasm_plugin_proto()?; make_pulsar_proto()?; diff --git a/agent/src/flow_generator/protocol_logs/mq/pulsar.rs b/agent/src/flow_generator/protocol_logs/mq/pulsar.rs index 1473dcf9033..2487ee71cb0 100644 --- a/agent/src/flow_generator/protocol_logs/mq/pulsar.rs +++ b/agent/src/flow_generator/protocol_logs/mq/pulsar.rs @@ -1,4 +1,5 @@ #[path = "pulsar.proto.rs"] +#[rustfmt::skip] mod pulsar_proto; use prost::Message; diff --git a/agent/src/flow_generator/protocol_logs/rpc/brpc.rs b/agent/src/flow_generator/protocol_logs/rpc/brpc.rs index d0a3311e829..b3185e32140 100644 --- a/agent/src/flow_generator/protocol_logs/rpc/brpc.rs +++ b/agent/src/flow_generator/protocol_logs/rpc/brpc.rs @@ -1,4 +1,5 @@ #[path = "brpc/brpc.policy.rs"] +#[rustfmt::skip] mod brpc_policy; use brpc_policy::RpcMeta; diff --git a/agent/src/platform/platform_synchronizer/linux.rs b/agent/src/platform/platform_synchronizer/linux.rs index 278270502f1..c2e17924197 100644 --- a/agent/src/platform/platform_synchronizer/linux.rs +++ b/agent/src/platform/platform_synchronizer/linux.rs @@ -35,7 +35,10 @@ use crate::{ utils::lru::Lru, }; use public::{ - proto::{agent, trident::{GpidSyncRequest, GpidSyncResponse}}, + proto::{ + agent, + trident::{GpidSyncRequest, GpidSyncResponse}, + }, queue::Receiver, }; diff --git a/agent/src/plugin/wasm/mod.rs b/agent/src/plugin/wasm/mod.rs index d8785696022..50d467034c8 100644 --- a/agent/src/plugin/wasm/mod.rs +++ b/agent/src/plugin/wasm/mod.rs @@ -106,6 +106,7 @@ */ #[path = "wasm_plugin.rs"] +#[rustfmt::skip] pub mod wasm_plugin; mod abi_export;