Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for artifact size profiling #87404

Merged
merged 2 commits into from
Oct 20, 2021
Merged

Add support for artifact size profiling #87404

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
947a33b
Add support for artifact size profiling
rylev Jul 23, 2021
757f76e
Update to measureme v10
rylev Oct 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2138,6 +2138,20 @@ dependencies = [
"smallvec",
]

[[package]]
name = "measureme"
version = "10.0.0"
source = "registry+/~https://github.com/rust-lang/crates.io-index"
checksum = "bd460fad6e55ca82fa0cd9dab0d315294188fd9ec6efbf4105e5635d4872ef9c"
dependencies = [
"log",
"memmap2",
"parking_lot",
"perf-event-open-sys",
"rustc-hash",
"smallvec",
]

[[package]]
name = "memchr"
version = "2.4.1"
Expand Down Expand Up @@ -2242,7 +2256,7 @@ dependencies = [
"hex 0.4.2",
"libc",
"log",
"measureme",
"measureme 9.1.2",
"rand 0.8.4",
"rustc-workspace-hack",
"rustc_version 0.4.0",
Expand Down Expand Up @@ -3219,7 +3233,7 @@ dependencies = [
"indexmap",
"jobserver",
"libc",
"measureme",
"measureme 9.1.2",
"memmap2",
"parking_lot",
"rustc-ap-rustc_graphviz",
Expand Down Expand Up @@ -3657,7 +3671,7 @@ dependencies = [
"bitflags",
"cstr",
"libc",
"measureme",
"measureme 10.0.0",
"rustc-demangle",
"rustc_arena",
"rustc_ast",
Expand Down Expand Up @@ -3752,7 +3766,7 @@ dependencies = [
"indexmap",
"jobserver",
"libc",
"measureme",
"measureme 10.0.0",
"memmap2",
"parking_lot",
"rustc-hash",
Expand Down Expand Up @@ -4276,7 +4290,7 @@ dependencies = [
name = "rustc_query_impl"
version = "0.0.0"
dependencies = [
"measureme",
"measureme 10.0.0",
"rustc-rayon-core",
"rustc_ast",
"rustc_data_structures",
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_llvm/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ doctest = false
bitflags = "1.0"
cstr = "0.2"
libc = "0.2"
measureme = "9.1.0"
measureme = "10.0.0"
snap = "1"
tracing = "0.1"
rustc_middle = { path = "../rustc_middle" }
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_data_structures/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ rustc-hash = "1.1.0"
smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
rustc_index = { path = "../rustc_index", package = "rustc_index" }
bitflags = "1.2.1"
measureme = "9.1.0"
measureme = "10.0.0"
libc = "0.2"
stacker = "0.1.14"
tempfile = "3.2"
Expand Down
43 changes: 40 additions & 3 deletions compiler/rustc_data_structures/src/profiling.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,14 @@ bitflags::bitflags! {
const FUNCTION_ARGS = 1 << 6;
const LLVM = 1 << 7;
const INCR_RESULT_HASHING = 1 << 8;
const ARTIFACT_SIZES = 1 << 9;

const DEFAULT = Self::GENERIC_ACTIVITIES.bits |
Self::QUERY_PROVIDERS.bits |
Self::QUERY_BLOCKED.bits |
Self::INCR_CACHE_LOADS.bits |
Self::INCR_RESULT_HASHING.bits;
Self::INCR_RESULT_HASHING.bits |
Self::ARTIFACT_SIZES.bits;

const ARGS = Self::QUERY_KEYS.bits | Self::FUNCTION_ARGS.bits;
}
Expand All @@ -136,6 +138,7 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[
("args", EventFilter::ARGS),
("llvm", EventFilter::LLVM),
("incr-result-hashing", EventFilter::INCR_RESULT_HASHING),
("artifact-sizes", EventFilter::ARTIFACT_SIZES),
];

/// Something that uniquely identifies a query invocation.
Expand Down Expand Up @@ -285,6 +288,33 @@ impl SelfProfilerRef {
})
}

/// Record the size of an artifact that the compiler produces
wesleywiser marked this conversation as resolved.
Show resolved Hide resolved
///
/// `artifact_kind` is the class of artifact (e.g., query_cache, object_file, etc.)
/// `artifact_name` is an identifier to the specific artifact being stored (usually a filename)
#[inline(always)]
pub fn artifact_size<A>(&self, artifact_kind: &str, artifact_name: A, size: u64)
jyn514 marked this conversation as resolved.
Show resolved Hide resolved
where
A: Borrow<str> + Into<String>,
{
drop(self.exec(EventFilter::ARTIFACT_SIZES, |profiler| {
let builder = EventIdBuilder::new(&profiler.profiler);
let event_label = profiler.get_or_alloc_cached_string(artifact_kind);
let event_arg = profiler.get_or_alloc_cached_string(artifact_name);
let event_id = builder.from_label_and_arg(event_label, event_arg);
let thread_id = get_thread_id();

profiler.profiler.record_integer_event(
profiler.artifact_size_event_kind,
event_id,
thread_id,
size,
);

TimingGuard::none()
}))
}

#[inline(always)]
pub fn generic_activity_with_args(
&self,
Expand Down Expand Up @@ -372,7 +402,7 @@ impl SelfProfilerRef {
) {
drop(self.exec(event_filter, |profiler| {
let event_id = StringId::new_virtual(query_invocation_id.0);
let thread_id = std::thread::current().id().as_u64().get() as u32;
let thread_id = get_thread_id();

profiler.profiler.record_instant_event(
event_kind(profiler),
Expand Down Expand Up @@ -425,6 +455,7 @@ pub struct SelfProfiler {
incremental_result_hashing_event_kind: StringId,
query_blocked_event_kind: StringId,
query_cache_hit_event_kind: StringId,
artifact_size_event_kind: StringId,
}

impl SelfProfiler {
Expand All @@ -447,6 +478,7 @@ impl SelfProfiler {
profiler.alloc_string("IncrementalResultHashing");
let query_blocked_event_kind = profiler.alloc_string("QueryBlocked");
let query_cache_hit_event_kind = profiler.alloc_string("QueryCacheHit");
let artifact_size_event_kind = profiler.alloc_string("ArtifactSize");

let mut event_filter_mask = EventFilter::empty();

Expand Down Expand Up @@ -491,6 +523,7 @@ impl SelfProfiler {
incremental_result_hashing_event_kind,
query_blocked_event_kind,
query_cache_hit_event_kind,
artifact_size_event_kind,
})
}

Expand Down Expand Up @@ -561,7 +594,7 @@ impl<'a> TimingGuard<'a> {
event_kind: StringId,
event_id: EventId,
) -> TimingGuard<'a> {
let thread_id = std::thread::current().id().as_u64().get() as u32;
let thread_id = get_thread_id();
let raw_profiler = &profiler.profiler;
let timing_guard =
raw_profiler.start_recording_interval_event(event_kind, event_id, thread_id);
Expand Down Expand Up @@ -655,6 +688,10 @@ pub fn duration_to_secs_str(dur: std::time::Duration) -> String {
format!("{:.3}", dur.as_secs_f64())
}

fn get_thread_id() -> u32 {
std::thread::current().id().as_u64().get() as u32
}

// Memory reporting
cfg_if! {
if #[cfg(windows)] {
Expand Down
6 changes: 6 additions & 0 deletions compiler/rustc_incremental/src/persist/file_format.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,12 @@ where
return;
}

sess.prof.artifact_size(
&name.replace(' ', "_"),
path_buf.file_name().unwrap().to_string_lossy(),
encoder.position() as u64,
);

debug!("save: data written to disk successfully");
}

Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_query_impl/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ edition = "2021"
doctest = false

[dependencies]
measureme = "9.0.0"
measureme = "10.0.0"
rustc-rayon-core = "0.3.1"
tracing = "0.1"
rustc_ast = { path = "../rustc_ast" }
Expand Down
10 changes: 7 additions & 3 deletions compiler/rustc_query_system/src/dep_graph/serialized.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ impl<K: DepKind> EncoderState<K> {
index
}

fn finish(self) -> FileEncodeResult {
fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
let Self { mut encoder, total_node_count, total_edge_count, result, stats: _ } = self;
let () = result?;

Expand All @@ -235,7 +235,11 @@ impl<K: DepKind> EncoderState<K> {
IntEncodedWithFixedSize(edge_count).encode(&mut encoder)?;
debug!("position: {:?}", encoder.position());
// Drop the encoder so that nothing is written after the counts.
encoder.flush()
let result = encoder.flush();
// FIXME(rylev): we hardcode the dep graph file name so we don't need a dependency on
// rustc_incremental just for that.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rustc_incremental already depends on rustc_query_impl (directly or through rustc_middle). You can export a const to be accessed by rustc_incremental.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that would actually introduce a circular dependency, right?

rustc_query_system -> rustc_incremental -> rustc_middle
         ^                                      |
         |                                      |
         +--------------------------------------+

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the failure mode here if the dep graph file name changes is just that we have the wrong file name in the self-profile data which doesn't seem important enough to add the dependency, so this seems fine.

profiler.artifact_size("dep_graph", "dep-graph.bin", encoder.position() as u64);
result
}
}

Expand Down Expand Up @@ -332,6 +336,6 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {

pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
self.status.into_inner().finish()
self.status.into_inner().finish(profiler)
}
}
2 changes: 1 addition & 1 deletion compiler/rustc_session/src/options.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1279,7 +1279,7 @@ options! {
"specify the events recorded by the self profiler;
for example: `-Z self-profile-events=default,query-keys`
all options: none, all, default, generic-activity, query-provider, query-cache-hit
query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm"),
query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"),
share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
"make the current crate share its generic instantiations"),
show_span: Option<String> = (None, parse_opt_string, [TRACKED],
Expand Down