Auto merge of rust-lang#101403 - bjorn3:dylib_lto, r=Mark-Simulacrum

Enable LTO for rustc_driver.so

Alternative to rust-lang#97154

This enables LTO'ing dylibs behind a feature flag and uses this feature for compiling rustc_driver.so.

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -32,8 +32,8 @@ pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
 
 pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
     match crate_type {
-        CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true,
-        CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false,
+        CrateType::Executable | CrateType::Dylib | CrateType::Staticlib | CrateType::Cdylib => true,
+        CrateType::Rlib | CrateType::ProcMacro => false,
     }
 }
 
@@ -73,17 +73,6 @@ fn prepare_lto(
     // with either fat or thin LTO
     let mut upstream_modules = Vec::new();
     if cgcx.lto != Lto::ThinLocal {
-        if cgcx.opts.cg.prefer_dynamic {
-            diag_handler
-                .struct_err("cannot prefer dynamic linking when performing LTO")
-                .note(
-                    "only 'staticlib', 'bin', and 'cdylib' outputs are \
-                               supported with LTO",
-                )
-                .emit();
-            return Err(FatalError);
-        }
-
         // Make sure we actually can run LTO
         for crate_type in cgcx.crate_types.iter() {
             if !crate_type_allows_lto(*crate_type) {
@@ -92,9 +81,25 @@ fn prepare_lto(
                                             static library outputs",
                 );
                 return Err(e);
+            } else if *crate_type == CrateType::Dylib {
+                if !cgcx.opts.unstable_opts.dylib_lto {
+                    return Err(diag_handler
+                        .fatal("lto cannot be used for `dylib` crate type without `-Zdylib-lto`"));
+                }
             }
         }
 
+        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
+            diag_handler
+                .struct_err("cannot prefer dynamic linking when performing LTO")
+                .note(
+                    "only 'staticlib', 'bin', and 'cdylib' outputs are \
+                               supported with LTO",
+                )
+                .emit();
+            return Err(FatalError);
+        }
+
         for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
             let exported_symbols =
                 cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -11,7 +11,7 @@ use rustc_metadata::find_native_static_library;
 use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
-use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip};
+use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Lto, Strip};
 use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SplitDwarfKind};
 use rustc_session::cstore::DllImport;
 use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename};
@@ -39,6 +39,7 @@ use cc::windows_registry;
 use regex::Regex;
 use tempfile::Builder as TempFileBuilder;
 
+use itertools::Itertools;
 use std::borrow::Borrow;
 use std::cell::OnceCell;
 use std::collections::BTreeSet;
@@ -208,11 +209,29 @@ pub fn link_binary<'a>(
 }
 
 pub fn each_linked_rlib(
+    sess: &Session,
     info: &CrateInfo,
     f: &mut dyn FnMut(CrateNum, &Path),
 ) -> Result<(), errors::LinkRlibError> {
     let crates = info.used_crates.iter();
     let mut fmts = None;
+
+    let lto_active = matches!(sess.lto(), Lto::Fat | Lto::Thin);
+    if lto_active {
+        for combination in info.dependency_formats.iter().combinations(2) {
+            let (ty1, list1) = &combination[0];
+            let (ty2, list2) = &combination[1];
+            if list1 != list2 {
+                return Err(errors::LinkRlibError::IncompatibleDependencyFormats {
+                    ty1: format!("{ty1:?}"),
+                    ty2: format!("{ty2:?}"),
+                    list1: format!("{list1:?}"),
+                    list2: format!("{list2:?}"),
+                });
+            }
+        }
+    }
+
     for (ty, list) in info.dependency_formats.iter() {
         match ty {
             CrateType::Executable
@@ -222,6 +241,10 @@ pub fn each_linked_rlib(
                 fmts = Some(list);
                 break;
             }
+            CrateType::Dylib if lto_active => {
+                fmts = Some(list);
+                break;
+            }
             _ => {}
         }
     }
@@ -490,7 +513,7 @@ fn link_staticlib<'a>(
     )?;
     let mut all_native_libs = vec![];
 
-    let res = each_linked_rlib(&codegen_results.crate_info, &mut |cnum, path| {
+    let res = each_linked_rlib(sess, &codegen_results.crate_info, &mut |cnum, path| {
         let name = codegen_results.crate_info.crate_name[&cnum];
         let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
 

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -13,7 +13,7 @@ use rustc_middle::ty::query::{ExternProviders, Providers};
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
 use rustc_middle::ty::Instance;
 use rustc_middle::ty::{self, SymbolName, TyCtxt};
-use rustc_session::config::CrateType;
+use rustc_session::config::{CrateType, OomStrategy};
 use rustc_target::spec::SanitizerSet;
 
 pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel {
@@ -206,6 +206,15 @@ fn exported_symbols_provider_local<'tcx>(
                 },
             ));
         }
+
+        symbols.push((
+            ExportedSymbol::NoDefId(SymbolName::new(tcx, OomStrategy::SYMBOL)),
+            SymbolExportInfo {
+                level: SymbolExportLevel::Rust,
+                kind: SymbolExportKind::Text,
+                used: false,
+            },
+        ));
     }
 
     if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() {

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -999,6 +999,14 @@ fn start_executing_work<B: ExtraBackendMethods>(
     let coordinator_send = tx_to_llvm_workers;
     let sess = tcx.sess;
 
+    let mut each_linked_rlib_for_lto = Vec::new();
+    drop(link::each_linked_rlib(sess, crate_info, &mut |cnum, path| {
+        if link::ignored_for_lto(sess, crate_info, cnum) {
+            return;
+        }
+        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
+    }));
+
     // Compute the set of symbols we need to retain when doing LTO (if we need to)
     let exported_symbols = {
         let mut exported_symbols = FxHashMap::default();
@@ -1020,7 +1028,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
             }
             Lto::Fat | Lto::Thin => {
                 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
-                for &cnum in tcx.crates(()).iter() {
+                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
                     exported_symbols.insert(cnum, copy_symbols(cnum));
                 }
                 Some(Arc::new(exported_symbols))
@@ -1040,14 +1048,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
         })
         .expect("failed to spawn helper thread");
 
-    let mut each_linked_rlib_for_lto = Vec::new();
-    drop(link::each_linked_rlib(crate_info, &mut |cnum, path| {
-        if link::ignored_for_lto(sess, crate_info, cnum) {
-            return;
-        }
-        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
-    }));
-
     let ol =
         if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
             // If we know that we won’t be doing codegen, create target machines without optimisation.

compiler/rustc_codegen_ssa/src/errors.rs

@@ -127,6 +127,9 @@ pub enum LinkRlibError {
 
     #[diag(codegen_ssa_rlib_not_found)]
     NotFound { crate_name: Symbol },
+
+    #[diag(codegen_ssa_rlib_incompatible_dependency_formats)]
+    IncompatibleDependencyFormats { ty1: String, ty2: String, list1: String, list2: String },
 }
 
 pub struct ThorinErrorWrapper(pub thorin::Error);

compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl

@@ -34,6 +34,8 @@ codegen_ssa_rlib_only_rmeta_found = could not find rlib for: `{$crate_name}`, fo
 
 codegen_ssa_rlib_not_found = could not find rlib for: `{$crate_name}`
 
+codegen_ssa_rlib_incompatible_dependency_formats = `{$ty1}` and `{$ty2}` do not have equivalent dependency formats (`{$list1}` vs `{$list2}`)
+
 codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status}
 
 codegen_ssa_extern_funcs_not_found = some `extern` functions couldn't be found; some native libraries may need to be installed or have their path specified

compiler/rustc_interface/src/tests.rs

@@ -648,6 +648,7 @@ fn test_unstable_options_tracking_hash() {
     untracked!(dump_mir_dir, String::from("abc"));
     untracked!(dump_mir_exclude_pass_number, true);
     untracked!(dump_mir_graphviz, true);
+    untracked!(dylib_lto, true);
     untracked!(emit_stack_sizes, true);
     untracked!(future_incompat_test, true);
     untracked!(hir_stats, true);

compiler/rustc_session/src/options.rs

@@ -1295,6 +1295,8 @@ options! {
         an additional `.html` file showing the computed coverage spans."),
     dwarf_version: Option<u32> = (None, parse_opt_number, [TRACKED],
         "version of DWARF debug information to emit (default: 2 or 4, depending on platform)"),
+    dylib_lto: bool = (false, parse_bool, [UNTRACKED],
+        "enables LTO for dylib crate type"),
     emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED],
         "emit a section containing stack size metadata (default: no)"),
     emit_thin_lto: bool = (true, parse_bool, [TRACKED],

config.toml.example

@@ -638,6 +638,11 @@ changelog-seen = 2
 # If an explicit setting is given, it will be used for all parts of the codebase.
 #new-symbol-mangling = true|false (see comment)
 
+# Select LTO mode that will be used for compiling rustc. By default, thin local LTO
+# (LTO within a single crate) is used (like for any Rust crate). You can also select
+# "thin" or "fat" to apply Thin/Fat LTO to the `rustc_driver` dylib.
+#lto = "thin-local"
+
 # =============================================================================
 # Options for specific targets
 #

src/bootstrap/compile.rs

@@ -21,7 +21,7 @@ use serde::Deserialize;
 use crate::builder::Cargo;
 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::cache::{Interned, INTERNER};
-use crate::config::{LlvmLibunwind, TargetSelection};
+use crate::config::{LlvmLibunwind, RustcLto, TargetSelection};
 use crate::dist;
 use crate::native;
 use crate::tool::SourceType;
@@ -701,6 +701,28 @@ impl Step for Rustc {
             ));
         }
 
+        // cfg(bootstrap): remove if condition once the bootstrap compiler supports dylib LTO
+        if compiler.stage != 0 {
+            match builder.config.rust_lto {
+                RustcLto::Thin | RustcLto::Fat => {
+                    // Since using LTO for optimizing dylibs is currently experimental,
+                    // we need to pass -Zdylib-lto.
+                    cargo.rustflag("-Zdylib-lto");
+                    // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when
+                    // compiling dylibs (and their dependencies), even when LTO is enabled for the
+                    // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here.
+                    let lto_type = match builder.config.rust_lto {
+                        RustcLto::Thin => "thin",
+                        RustcLto::Fat => "fat",
+                        _ => unreachable!(),
+                    };
+                    cargo.rustflag(&format!("-Clto={}", lto_type));
+                    cargo.rustflag("-Cembed-bitcode=yes");
+                }
+                RustcLto::ThinLocal => { /* Do nothing, this is the default */ }
+            }
+        }
+
         builder.info(&format!(
             "Building stage{} compiler artifacts ({} -> {})",
             compiler.stage, &compiler.host, target

src/bootstrap/config.rs

@@ -158,6 +158,7 @@ pub struct Config {
     pub rust_new_symbol_mangling: Option<bool>,
     pub rust_profile_use: Option<String>,
     pub rust_profile_generate: Option<String>,
+    pub rust_lto: RustcLto,
     pub llvm_profile_use: Option<String>,
     pub llvm_profile_generate: bool,
     pub llvm_libunwind_default: Option<LlvmLibunwind>,
@@ -319,6 +320,28 @@ impl SplitDebuginfo {
     }
 }
 
+/// LTO mode used for compiling rustc itself.
+#[derive(Default, Clone)]
+pub enum RustcLto {
+    #[default]
+    ThinLocal,
+    Thin,
+    Fat,
+}
+
+impl std::str::FromStr for RustcLto {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "thin-local" => Ok(RustcLto::ThinLocal),
+            "thin" => Ok(RustcLto::Thin),
+            "fat" => Ok(RustcLto::Fat),
+            _ => Err(format!("Invalid value for rustc LTO: {}", s)),
+        }
+    }
+}
+
 #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct TargetSelection {
     pub triple: Interned<String>,
@@ -726,6 +749,7 @@ define_config! {
         profile_use: Option<String> = "profile-use",
         // ignored; this is set from an env var set by bootstrap.py
         download_rustc: Option<StringOrBool> = "download-rustc",
+        lto: Option<String> = "lto",
     }
 }
 
@@ -1173,6 +1197,12 @@ impl Config {
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
             config.download_rustc_commit = download_ci_rustc_commit(&config, rust.download_rustc);
+
+            config.rust_lto = rust
+                .lto
+                .as_deref()
+                .map(|value| RustcLto::from_str(value).unwrap())
+                .unwrap_or_default();
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -78,7 +78,8 @@ ENV RUST_CONFIGURE_ARGS \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
       --set rust.jemalloc \
-      --set rust.use-lld=true
+      --set rust.use-lld=true \
+      --set rust.lto=thin
 ENV SCRIPT ../src/ci/pgo.sh python3 ../x.py dist \
     --host $HOSTS --target $HOSTS \
     --include-default-paths \

src/doc/unstable-book/src/compiler-flags/dylib-lto.md

@@ -0,0 +1,4 @@
+## `dylib-lto`
+
+This option enables using LTO for the `dylib` crate type. This is currently only used for compiling
+`rustc` itself (more specifically, the `librustc_driver` dylib).

src/test/rustdoc-ui/z-help.stdout

@@ -36,6 +36,7 @@
     -Z                       dump-mir-graphviz=val -- in addition to `.mir` files, create graphviz `.dot` files (and with `-Z instrument-coverage`, also create a `.dot` file for the MIR-derived coverage graph) (default: no)
     -Z                       dump-mir-spanview=val -- in addition to `.mir` files, create `.html` files to view spans for all `statement`s (including terminators), only `terminator` spans, or computed `block` spans (one span encompassing a block's terminator and all statements). If `-Z instrument-coverage` is also enabled, create an additional `.html` file showing the computed coverage spans.
     -Z                           dwarf-version=val -- version of DWARF debug information to emit (default: 2 or 4, depending on platform)
+    -Z                               dylib-lto=val -- enables LTO for dylib crate type
     -Z                        emit-stack-sizes=val -- emit a section containing stack size metadata (default: no)
     -Z                           emit-thin-lto=val -- emit the bc module with thin LTO info (default: yes)
     -Z               export-executable-symbols=val -- export symbols from executables, as if they were dynamic libraries