diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs index afcf8b15cd800..ab1a984c918ee 100644 --- a/compiler/rustc_ast_lowering/src/format.rs +++ b/compiler/rustc_ast_lowering/src/format.rs @@ -380,7 +380,6 @@ fn expand_format_args<'hir>( } } })); - let lit_pieces = ctx.expr_array_ref(fmt.span, lit_pieces); // Whether we'll use the `Arguments::new_v1_formatted` form (true), // or the `Arguments::new_v1` form (false). @@ -407,6 +406,24 @@ fn expand_format_args<'hir>( } } + let arguments = fmt.arguments.all_args(); + + if allow_const && lit_pieces.len() <= 1 && arguments.is_empty() && argmap.is_empty() { + // Generate: + // ::new_str(literal) + let new = ctx.arena.alloc(ctx.expr_lang_item_type_relative( + macsp, + hir::LangItem::FormatArguments, + sym::new_str, + )); + let args = if lit_pieces.is_empty() { + ctx.arena.alloc_from_iter([ctx.expr_str(fmt.span, kw::Empty)]) // Empty string. + } else { + lit_pieces // Just one single literal string piece. + }; + return hir::ExprKind::Call(new, args); + } + let format_options = use_format_options.then(|| { // Generate: // &[format_spec_0, format_spec_1, format_spec_2] @@ -421,20 +438,6 @@ fn expand_format_args<'hir>( ctx.expr_array_ref(macsp, ctx.arena.alloc_from_iter(elements)) }); - let arguments = fmt.arguments.all_args(); - - if allow_const && arguments.is_empty() && argmap.is_empty() { - // Generate: - // ::new_const(lit_pieces) - let new = ctx.arena.alloc(ctx.expr_lang_item_type_relative( - macsp, - hir::LangItem::FormatArguments, - sym::new_const, - )); - let new_args = ctx.arena.alloc_from_iter([lit_pieces]); - return hir::ExprKind::Call(new, new_args); - } - // If the args array contains exactly all the original arguments once, // in order, we can use a simple array instead of a `match` construction. // However, if there's a yield point in any argument except the first one, @@ -555,6 +558,8 @@ fn expand_format_args<'hir>( ) }; + let lit_pieces = ctx.expr_array_ref(fmt.span, lit_pieces); + if let Some(format_options) = format_options { // Generate: // ::new_v1_formatted( diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 448314cd9e113..3b5b23fbd7f46 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1030,6 +1030,7 @@ symbols! { new_lower_hex, new_octal, new_pointer, + new_str, new_unchecked, new_upper_exp, new_upper_hex, diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs index 9ce6093f1d1f3..4ff8218a350ee 100644 --- a/library/core/src/fmt/mod.rs +++ b/library/core/src/fmt/mod.rs @@ -7,9 +7,11 @@ use crate::char::EscapeDebugExtArgs; use crate::iter; use crate::marker::PhantomData; use crate::mem; -use crate::num::fmt as numfmt; +use crate::num::{fmt as numfmt, NonZeroUsize}; use crate::ops::Deref; +use crate::ptr; use crate::result; +use crate::slice; use crate::str; mod builders; @@ -277,55 +279,111 @@ impl<'a> Formatter<'a> { #[stable(feature = "rust1", since = "1.0.0")] #[derive(Copy, Clone)] pub struct Arguments<'a> { - // Format string pieces to print. - pieces: &'a [&'static str], + /// The number of string pieces and place holders combined. + /// + /// For example: + /// - 1 for `format_args!("abc")` + /// - 2 for `format_args!("abc{}")` + /// - 3 for `format_args!("abc{}xyz")` + /// - 4 for `format_args!("abc{}xyz{}")` + /// - 5 for `format_args!("abc{}xyz{}123")` + /// + /// The first part is always a string piece, but it may be an empty string. + /// E.g. format_args!("{}") has two parts, one empty string piece and one placeholder. + /// + /// The number of placeholders is `num_parts / 2`. + /// The number of string pieces is `(num_parts + 1) / 2`. + num_parts: NonZeroUsize, + + /// The string pieces and the placeholders. + /// + /// If `num_parts` is one, this stores the &'static str directly. + /// Otherwise, it stores pointers to both the slice of string pieces and the slice of placeholders. + /// + /// The length of those slices are determined by the `num_parts` field above. + parts: Parts, - // Placeholder specs, or `None` if all specs are default (as in "{}{}"). - fmt: Option<&'a [rt::Placeholder]>, + /// Pointer to the start of the array of arguments. + args: *const rt::Argument<'a>, +} - // Dynamic arguments for interpolation, to be interleaved with string - // pieces. (Every argument is preceded by a string piece.) - args: &'a [rt::Argument<'a>], +#[derive(Copy, Clone)] +union Parts { + /// Used if `num_parts == 1`. + string: &'static str, + /// Used if `num_parts > 1`. + /// + /// The placaeholders pointer can be null for default placeholders: + /// a placeholder for each argument once, in order, with default formatting options. + strings_and_placeholders: (*const &'static str, *const rt::Placeholder), } +// `Arguments` will never be Send nor Sync, because it may borrow arguments that are not Sync. +#[stable(feature = "rust1", since = "1.0.0")] +impl !Sync for Arguments<'_> {} +#[stable(feature = "rust1", since = "1.0.0")] +impl !Send for Arguments<'_> {} + /// Used by the format_args!() macro to create a fmt::Arguments object. #[doc(hidden)] #[unstable(feature = "fmt_internals", issue = "none")] impl<'a> Arguments<'a> { #[inline] #[rustc_const_unstable(feature = "const_fmt_arguments_new", issue = "none")] - pub const fn new_const(pieces: &'a [&'static str]) -> Self { - if pieces.len() > 1 { - panic!("invalid args"); + #[cfg(bootstrap)] + pub const fn new_const(strings: &'a [&'static str]) -> Self { + match strings { + [] => Self::new_str(""), + [s] => Self::new_str(s), + _ => panic!("invalid args"), } - Arguments { pieces, fmt: None, args: &[] } + } + + #[inline] + #[rustc_const_unstable(feature = "const_fmt_arguments_new", issue = "none")] + pub const fn new_str(s: &'static str) -> Self { + Self { num_parts: NonZeroUsize::MIN, parts: Parts { string: s }, args: ptr::null() } } /// When using the format_args!() macro, this function is used to generate the /// Arguments structure. #[inline] - pub fn new_v1(pieces: &'a [&'static str], args: &'a [rt::Argument<'a>]) -> Arguments<'a> { - if pieces.len() < args.len() || pieces.len() > args.len() + 1 { - panic!("invalid args"); + pub fn new_v1(strings: &'a [&'static str], args: &'a [rt::Argument<'a>]) -> Self { + // The number of strings and args should be the same, + // except there may be one additional string after the last arg. + assert!(strings.len() == args.len() || strings.len() == args.len() + 1, "invalid args"); + match NonZeroUsize::new(strings.len() + args.len()) { + None => Self::new_str(""), + Some(NonZeroUsize::MIN) => Self::new_str(strings[0]), + Some(num_parts) => Self { + num_parts, + parts: Parts { strings_and_placeholders: (strings.as_ptr(), ptr::null()) }, + args: args.as_ptr(), + }, } - Arguments { pieces, fmt: None, args } } /// This function is used to specify nonstandard formatting parameters. /// /// An `rt::UnsafeArg` is required because the following invariants must be held /// in order for this function to be safe: - /// 1. The `pieces` slice must be at least as long as `fmt`. - /// 2. Every `rt::Placeholder::position` value within `fmt` must be a valid index of `args`. - /// 3. Every `rt::Count::Param` within `fmt` must contain a valid index of `args`. + /// 1. `placeholders` must be nonempty. + /// 2. The `strings` slice must be at least as long as `placeholders`. + /// 3. Every `rt::Placeholder::position` value within `placeholders` must be a valid index of `args`. + /// 4. Every `rt::Count::Param` within `placeholders` must contain a valid index of `args`. #[inline] pub fn new_v1_formatted( - pieces: &'a [&'static str], + strings: &'a [&'static str], args: &'a [rt::Argument<'a>], - fmt: &'a [rt::Placeholder], + placeholders: &'a [rt::Placeholder], _unsafe_arg: rt::UnsafeArg, - ) -> Arguments<'a> { - Arguments { pieces, fmt: Some(fmt), args } + ) -> Self { + Self { + // SAFETY: The caller must guarantee `placeholders` is nonempty. + num_parts: unsafe { NonZeroUsize::new_unchecked(strings.len() + placeholders.len()) }, + parts: Parts { strings_and_placeholders: (strings.as_ptr(), placeholders.as_ptr()) }, + args: args.as_ptr(), + } } /// Estimates the length of the formatted text. @@ -334,22 +392,37 @@ impl<'a> Arguments<'a> { /// when using `format!`. Note: this is neither the lower nor upper bound. #[inline] pub fn estimated_capacity(&self) -> usize { - let pieces_length: usize = self.pieces.iter().map(|x| x.len()).sum(); - - if self.args.is_empty() { - pieces_length - } else if !self.pieces.is_empty() && self.pieces[0].is_empty() && pieces_length < 16 { - // If the format string starts with an argument, - // don't preallocate anything, unless length - // of pieces is significant. - 0 + let num_parts = self.num_parts.get(); + + if num_parts == 1 { + // SAFETY: With num_parts == 1, the `parts` field stores just the string. + unsafe { self.parts.string }.len() } else { - // There are some arguments, so any additional push - // will reallocate the string. To avoid that, - // we're "pre-doubling" the capacity here. - pieces_length.checked_mul(2).unwrap_or(0) + // SAFETY: With num_parts > 1, the `parts` field stores the pointers to the strings and + // placeholder slices. + let strings = unsafe { + slice::from_raw_parts(self.parts.strings_and_placeholders.0, (num_parts + 1) / 2) + }; + let strings_length: usize = strings.iter().map(|s| s.len()).sum(); + if strings[0].is_empty() && strings_length < 16 { + // If the format string starts with an argument, + // don't preallocate anything, unless length + // of strings is significant. + 0 + } else { + // There are some arguments, so any additional push + // will reallocate the string. To avoid that, + // we're "pre-doubling" the capacity here. + strings_length.checked_mul(2).unwrap_or(0) + } } } + + #[inline(always)] + unsafe fn arg(&self, n: usize) -> &rt::Argument<'a> { + // SAFETY: Caller needs to privde a valid index. + unsafe { &*self.args.add(n) } + } } impl<'a> Arguments<'a> { @@ -398,12 +471,13 @@ impl<'a> Arguments<'a> { #[stable(feature = "fmt_as_str", since = "1.52.0")] #[rustc_const_unstable(feature = "const_arguments_as_str", issue = "103900")] #[must_use] - #[inline] + #[inline(always)] pub const fn as_str(&self) -> Option<&'static str> { - match (self.pieces, self.args) { - ([], []) => Some(""), - ([s], []) => Some(s), - _ => None, + if self.num_parts.get() == 1 { + // SAFETY: With num_parts == 1, the `parts` field stores just the string. + Some(unsafe { self.parts.string }) + } else { + None } } } @@ -1077,80 +1151,70 @@ pub trait UpperExp { /// /// [`write!`]: crate::write! #[stable(feature = "rust1", since = "1.0.0")] -pub fn write(output: &mut dyn Write, args: Arguments<'_>) -> Result { +pub fn write(output: &mut dyn Write, fmt: Arguments<'_>) -> Result { let mut formatter = Formatter::new(output); - let mut idx = 0; - - match args.fmt { - None => { - // We can use default formatting parameters for all arguments. - for (i, arg) in args.args.iter().enumerate() { - // SAFETY: args.args and args.pieces come from the same Arguments, - // which guarantees the indexes are always within bounds. - let piece = unsafe { args.pieces.get_unchecked(i) }; - if !piece.is_empty() { - formatter.buf.write_str(*piece)?; - } - arg.fmt(&mut formatter)?; - idx += 1; + + if let Some(s) = fmt.as_str() { + return formatter.buf.write_str(s); + } + + // SAFETY: Since as_str() returned None, we know that `fmt.parts` contains the + // strings and placeholders pointers. + let (strings, placeholders) = unsafe { fmt.parts.strings_and_placeholders }; + + // Iterate over all parts (string, placeholder, string, ...). + // Even numbered parts are strings, odd numbered parts are placeholders. + for i in 0..fmt.num_parts.get() { + if i % 2 == 0 { + // SAFETY: The Arguments type guarantees the indexes are always within bounds. + let string = unsafe { &*strings.add(i / 2) }; + if !string.is_empty() { + formatter.buf.write_str(string)?; } - } - Some(fmt) => { - // Every spec has a corresponding argument that is preceded by - // a string piece. - for (i, arg) in fmt.iter().enumerate() { - // SAFETY: fmt and args.pieces come from the same Arguments, - // which guarantees the indexes are always within bounds. - let piece = unsafe { args.pieces.get_unchecked(i) }; - if !piece.is_empty() { - formatter.buf.write_str(*piece)?; - } - // SAFETY: arg and args.args come from the same Arguments, - // which guarantees the indexes are always within bounds. - unsafe { run(&mut formatter, arg, args.args) }?; - idx += 1; + } else { + if placeholders.is_null() { + // Use default placeholders: each argument once, in order, with default formatting. + // SAFETY: The Arguments type guarantees the indexes are always within bounds. + unsafe { fmt.arg(i / 2) }.fmt(&mut formatter)?; + } else { + // SAFETY: The Arguments type guarantees the indexes are always within bounds. + unsafe { run(&mut formatter, &fmt, &*placeholders.add(i / 2)) }?; } } } - // There can be only one trailing string piece left. - if let Some(piece) = args.pieces.get(idx) { - formatter.buf.write_str(*piece)?; - } - Ok(()) } -unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::Placeholder, args: &[rt::Argument<'_>]) -> Result { - fmt.fill = arg.fill; - fmt.align = arg.align; - fmt.flags = arg.flags; - // SAFETY: arg and args come from the same Arguments, - // which guarantees the indexes are always within bounds. +unsafe fn run( + out: &mut Formatter<'_>, + fmt: &Arguments<'_>, + placeholder: &rt::Placeholder, +) -> Result { + out.fill = placeholder.fill; + out.align = placeholder.align; + out.flags = placeholder.flags; + + // SAFETY: The Arguments type guarantees the indexes are always within bounds. unsafe { - fmt.width = getcount(args, &arg.width); - fmt.precision = getcount(args, &arg.precision); + out.width = getcount(fmt, &placeholder.width); + out.precision = getcount(fmt, &placeholder.precision); } - // Extract the correct argument - debug_assert!(arg.position < args.len()); - // SAFETY: arg and args come from the same Arguments, - // which guarantees its index is always within bounds. - let value = unsafe { args.get_unchecked(arg.position) }; + // SAFETY: The Arguments type guarantees the indexes are always within bounds. + let arg = unsafe { fmt.arg(placeholder.position) }; - // Then actually do some printing - value.fmt(fmt) + arg.fmt(out) } -unsafe fn getcount(args: &[rt::Argument<'_>], cnt: &rt::Count) -> Option { +unsafe fn getcount(fmt: &Arguments<'_>, cnt: &rt::Count) -> Option { match *cnt { rt::Count::Is(n) => Some(n), rt::Count::Implied => None, rt::Count::Param(i) => { - debug_assert!(i < args.len()); - // SAFETY: cnt and args come from the same Arguments, - // which guarantees this index is always within bounds. - unsafe { args.get_unchecked(i).as_usize() } + // SAFETY: The Arguments type guarantees the indexes are always within bounds, + // and the caller must give a `Count` from this same `Arguments` object. + unsafe { fmt.arg(i).as_usize() } } } } diff --git a/library/core/src/num/nonzero.rs b/library/core/src/num/nonzero.rs index 7f8d673c179a3..52eaa39200bc4 100644 --- a/library/core/src/num/nonzero.rs +++ b/library/core/src/num/nonzero.rs @@ -101,7 +101,7 @@ macro_rules! nonzero_integers { /// Returns the value as a primitive type. #[$stability] - #[inline] + #[inline(always)] #[rustc_const_stable(feature = "const_nonzero_get", since = "1.34.0")] pub const fn get(self) -> $Int { self.0 diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs index 281f8c1e1663c..0266c2f920005 100644 --- a/library/core/src/panicking.rs +++ b/library/core/src/panicking.rs @@ -124,7 +124,7 @@ pub const fn panic(expr: &'static str) -> ! { // truncation and padding (even though none is used here). Using // Arguments::new_v1 may allow the compiler to omit Formatter::pad from the // output binary, saving up to a few kilobytes. - panic_fmt(fmt::Arguments::new_const(&[expr])); + panic_fmt(fmt::Arguments::new_str(expr)); } /// Like `panic`, but without unwinding and track_caller to reduce the impact on codesize. @@ -133,7 +133,7 @@ pub const fn panic(expr: &'static str) -> ! { #[lang = "panic_nounwind"] // needed by codegen for non-unwinding panics #[rustc_nounwind] pub fn panic_nounwind(expr: &'static str) -> ! { - panic_nounwind_fmt(fmt::Arguments::new_const(&[expr]), /* force_no_backtrace */ false); + panic_nounwind_fmt(fmt::Arguments::new_str(expr), /* force_no_backtrace */ false); } /// Like `panic_nounwind`, but also inhibits showing a backtrace. @@ -141,7 +141,7 @@ pub fn panic_nounwind(expr: &'static str) -> ! { #[cfg_attr(feature = "panic_immediate_abort", inline)] #[rustc_nounwind] pub fn panic_nounwind_nobacktrace(expr: &'static str) -> ! { - panic_nounwind_fmt(fmt::Arguments::new_const(&[expr]), /* force_no_backtrace */ true); + panic_nounwind_fmt(fmt::Arguments::new_str(expr), /* force_no_backtrace */ true); } #[inline] diff --git a/tests/incremental/hashes/inherent_impls.rs b/tests/incremental/hashes/inherent_impls.rs index 285f857c9cbcf..c3b8e99777bc3 100644 --- a/tests/incremental/hashes/inherent_impls.rs +++ b/tests/incremental/hashes/inherent_impls.rs @@ -73,12 +73,12 @@ impl Foo { impl Foo { //------------ //--------------- - //------------------------------------------------------------ + //----------------------------------------------- // //-------------------------- //------------ //--------------- - //------------------------------------------------------------ + //----------------------------------------------- // //-------------------------- #[inline] @@ -95,12 +95,12 @@ impl Foo { impl Foo { #[rustc_clean( cfg="cfail2", - except="hir_owner_nodes,optimized_mir,promoted_mir,typeck" + except="hir_owner_nodes,optimized_mir,typeck" )] #[rustc_clean(cfg="cfail3")] #[rustc_clean( cfg="cfail5", - except="hir_owner_nodes,optimized_mir,promoted_mir,typeck" + except="hir_owner_nodes,optimized_mir,typeck" )] #[rustc_clean(cfg="cfail6")] #[inline] diff --git a/tests/incremental/hygiene/auxiliary/cached_hygiene.rs b/tests/incremental/hygiene/auxiliary/cached_hygiene.rs index b31f60e972bf0..676324943ede3 100644 --- a/tests/incremental/hygiene/auxiliary/cached_hygiene.rs +++ b/tests/incremental/hygiene/auxiliary/cached_hygiene.rs @@ -13,7 +13,7 @@ macro_rules! first_macro { } } -#[rustc_clean(except="hir_owner_nodes,typeck,optimized_mir,promoted_mir", cfg="rpass2")] +#[rustc_clean(except="hir_owner_nodes,typeck,optimized_mir", cfg="rpass2")] #[inline(always)] pub fn changed_fn() { // This will cause additional hygiene to be generate, diff --git a/tests/pretty/issue-4264.pp b/tests/pretty/issue-4264.pp index 4020a433d6254..83e608c3889dd 100644 --- a/tests/pretty/issue-4264.pp +++ b/tests/pretty/issue-4264.pp @@ -32,11 +32,10 @@ ({ let res = ((::alloc::fmt::format as - for<'a> fn(Arguments<'a>) -> String {format})(((<#[lang = "format_arguments"]>::new_const + for<'a> fn(Arguments<'a>) -> String {format})(((<#[lang = "format_arguments"]>::new_str as - fn(&[&'static str]) -> Arguments<'_> {Arguments::<'_>::new_const})((&([("test" - as &str)] as [&str; 1]) as &[&str; 1])) as Arguments<'_>)) - as String); + fn(&'static str) -> Arguments<'_> {Arguments::<'_>::new_str})(("test" + as &str)) as Arguments<'_>)) as String); (res as String) } as String); } as ()) diff --git a/tests/ui/borrowck/issue-64453.stderr b/tests/ui/borrowck/issue-64453.stderr index f032ea779dd48..6760ee9c2aaed 100644 --- a/tests/ui/borrowck/issue-64453.stderr +++ b/tests/ui/borrowck/issue-64453.stderr @@ -1,4 +1,4 @@ -error: `Arguments::<'a>::new_const` is not yet stable as a const fn +error: `Arguments::<'a>::new_str` is not yet stable as a const fn --> $DIR/issue-64453.rs:4:31 | LL | static settings_dir: String = format!(""); diff --git a/tests/ui/fmt/send-sync.stderr b/tests/ui/fmt/send-sync.stderr index e3ebe6cdcb81a..dff9b23ba677c 100644 --- a/tests/ui/fmt/send-sync.stderr +++ b/tests/ui/fmt/send-sync.stderr @@ -1,41 +1,27 @@ -error[E0277]: `core::fmt::rt::Opaque` cannot be shared between threads safely +error[E0277]: `Arguments<'_>` cannot be sent between threads safely --> $DIR/send-sync.rs:8:10 | LL | send(format_args!("{:?}", c)); - | ---- ^^^^^^^^^^^^^^^^^^^^^^^ `core::fmt::rt::Opaque` cannot be shared between threads safely + | ---- ^^^^^^^^^^^^^^^^^^^^^^^ `Arguments<'_>` cannot be sent between threads safely | | | required by a bound introduced by this call | - = help: within `[core::fmt::rt::Argument<'_>]`, the trait `Sync` is not implemented for `core::fmt::rt::Opaque` - = note: required because it appears within the type `&core::fmt::rt::Opaque` -note: required because it appears within the type `Argument<'_>` - --> $SRC_DIR/core/src/fmt/rt.rs:LL:COL - = note: required because it appears within the type `[Argument<'_>]` - = note: required for `&[core::fmt::rt::Argument<'_>]` to implement `Send` -note: required because it appears within the type `Arguments<'_>` - --> $SRC_DIR/core/src/fmt/mod.rs:LL:COL + = help: the trait `Send` is not implemented for `Arguments<'_>` note: required by a bound in `send` --> $DIR/send-sync.rs:1:12 | LL | fn send(_: T) {} | ^^^^ required by this bound in `send` -error[E0277]: `core::fmt::rt::Opaque` cannot be shared between threads safely +error[E0277]: `Arguments<'_>` cannot be shared between threads safely --> $DIR/send-sync.rs:9:10 | LL | sync(format_args!("{:?}", c)); - | ---- ^^^^^^^^^^^^^^^^^^^^^^^ `core::fmt::rt::Opaque` cannot be shared between threads safely + | ---- ^^^^^^^^^^^^^^^^^^^^^^^ `Arguments<'_>` cannot be shared between threads safely | | | required by a bound introduced by this call | - = help: within `Arguments<'_>`, the trait `Sync` is not implemented for `core::fmt::rt::Opaque` - = note: required because it appears within the type `&core::fmt::rt::Opaque` -note: required because it appears within the type `Argument<'_>` - --> $SRC_DIR/core/src/fmt/rt.rs:LL:COL - = note: required because it appears within the type `[Argument<'_>]` - = note: required because it appears within the type `&[Argument<'_>]` -note: required because it appears within the type `Arguments<'_>` - --> $SRC_DIR/core/src/fmt/mod.rs:LL:COL + = help: the trait `Sync` is not implemented for `Arguments<'_>` note: required by a bound in `sync` --> $DIR/send-sync.rs:2:12 |