Pass ParserOptions to the parser (#16220)

## Summary

This is part of the preparation for detecting syntax errors in the
parser from #16090. As suggested
in [this
comment](#16090),
I started working on a `ParseOptions` struct that could be stored in the
parser. For this initial refactor, I only made it hold the existing
`Mode` option, but for syntax errors, we will also need it to have a
`PythonVersion`. For that use case, I'm picturing something like a
`ParseOptions::with_python_version` method, so you can extend the
current calls to something like

```rust
ParseOptions::from(mode).with_python_version(settings.target_version)
```

But I thought it was worth adding `ParseOptions` alone without changing
any other behavior first.

Most of the diff is just updating call sites taking `Mode` to take
`ParseOptions::from(Mode)` or those taking `PySourceType`s to take
`ParseOptions::from(PySourceType)`. The interesting changes are in the
new `parser/options.rs` file and smaller parts of `parser/mod.rs` and
`ruff_python_parser/src/lib.rs`.

## Test Plan

Existing tests, this should not change any behavior.

crates/ruff_benchmark/benches/formatter.rs

@@ -8,7 +8,7 @@ use ruff_benchmark::{
     TestCase, LARGE_DATASET, NUMPY_CTYPESLIB, NUMPY_GLOBALS, PYDANTIC_TYPES, UNICODE_PYPINYIN,
 };
 use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions};
-use ruff_python_parser::{parse, Mode};
+use ruff_python_parser::{parse, Mode, ParseOptions};
 use ruff_python_trivia::CommentRanges;
 
 #[cfg(target_os = "windows")]
@@ -48,8 +48,8 @@ fn benchmark_formatter(criterion: &mut Criterion) {
             &case,
             |b, case| {
                 // Parse the source.
-                let parsed =
-                    parse(case.code(), Mode::Module).expect("Input should be a valid Python code");
+                let parsed = parse(case.code(), ParseOptions::from(Mode::Module))
+                    .expect("Input should be a valid Python code");
 
                 let comment_ranges = CommentRanges::from(parsed.tokens());
 

crates/ruff_dev/src/print_ast.rs

@@ -7,7 +7,7 @@ use anyhow::Result;
 
 use ruff_linter::source_kind::SourceKind;
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{parse, AsMode};
+use ruff_python_parser::{parse, ParseOptions};
 
 #[derive(clap::Args)]
 pub(crate) struct Args {
@@ -24,7 +24,8 @@ pub(crate) fn main(args: &Args) -> Result<()> {
             args.file.display()
         )
     })?;
-    let python_ast = parse(source_kind.source_code(), source_type.as_mode())?.into_syntax();
+    let python_ast =
+        parse(source_kind.source_code(), ParseOptions::from(source_type))?.into_syntax();
     println!("{python_ast:#?}");
     Ok(())
 }

crates/ruff_graph/src/lib.rs

@@ -4,7 +4,7 @@ use anyhow::Result;
 
 use ruff_db::system::{SystemPath, SystemPathBuf};
 use ruff_python_ast::helpers::to_module_path;
-use ruff_python_parser::{parse, Mode};
+use ruff_python_parser::{parse, Mode, ParseOptions};
 
 use crate::collector::Collector;
 pub use crate::db::ModuleDb;
@@ -30,7 +30,7 @@ impl ModuleImports {
     ) -> Result<Self> {
         // Read and parse the source code.
         let source = std::fs::read_to_string(path)?;
-        let parsed = parse(&source, Mode::Module)?;
+        let parsed = parse(&source, ParseOptions::from(Mode::Module))?;
 
         let module_path =
             package.and_then(|package| to_module_path(package.as_std_path(), path.as_std_path()));

crates/ruff_linter/src/message/mod.rs

@@ -309,7 +309,7 @@ mod tests {
 
     use ruff_diagnostics::{Diagnostic, DiagnosticKind, Edit, Fix};
     use ruff_notebook::NotebookIndex;
-    use ruff_python_parser::{parse_unchecked, Mode};
+    use ruff_python_parser::{parse_unchecked, Mode, ParseOptions};
     use ruff_source_file::{OneIndexed, SourceFileBuilder};
     use ruff_text_size::{Ranged, TextRange, TextSize};
 
@@ -325,7 +325,7 @@ if call(foo
 ";
         let locator = Locator::new(source);
         let source_file = SourceFileBuilder::new("syntax_errors.py", source).finish();
-        parse_unchecked(source, Mode::Module)
+        parse_unchecked(source, ParseOptions::from(Mode::Module))
             .errors()
             .iter()
             .map(|parse_error| {

crates/ruff_python_ast_integration_tests/tests/source_order.rs

@@ -4,7 +4,7 @@ use insta::assert_snapshot;
 
 use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal};
 use ruff_python_ast::{AnyNodeRef, BoolOp, CmpOp, Operator, Singleton, UnaryOp};
-use ruff_python_parser::{parse, Mode};
+use ruff_python_parser::{parse, Mode, ParseOptions};
 
 #[test]
 fn function_arguments() {
@@ -147,7 +147,7 @@ fn f_strings() {
 }
 
 fn trace_source_order_visitation(source: &str) -> String {
-    let parsed = parse(source, Mode::Module).unwrap();
+    let parsed = parse(source, ParseOptions::from(Mode::Module)).unwrap();
 
     let mut visitor = RecordVisitor::default();
     visitor.visit_mod(parsed.syntax());

crates/ruff_python_ast_integration_tests/tests/visitor.rs

@@ -13,7 +13,7 @@ use ruff_python_ast::{
     Expr, FString, FStringElement, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern,
     Stmt, StringLiteral, TypeParam, UnaryOp, WithItem,
 };
-use ruff_python_parser::{parse, Mode};
+use ruff_python_parser::{parse, Mode, ParseOptions};
 
 #[test]
 fn function_arguments() {
@@ -156,7 +156,7 @@ fn f_strings() {
 }
 
 fn trace_visitation(source: &str) -> String {
-    let parsed = parse(source, Mode::Module).unwrap();
+    let parsed = parse(source, ParseOptions::from(Mode::Module)).unwrap();
 
     let mut visitor = RecordVisitor::default();
     walk_module(&mut visitor, parsed.syntax());

crates/ruff_python_codegen/src/generator.rs

@@ -1435,7 +1435,7 @@ impl<'a> Generator<'a> {
 #[cfg(test)]
 mod tests {
     use ruff_python_ast::{Mod, ModModule};
-    use ruff_python_parser::{self, parse_module, Mode};
+    use ruff_python_parser::{self, parse_module, Mode, ParseOptions};
     use ruff_source_file::LineEnding;
 
     use crate::stylist::Indentation;
@@ -1467,7 +1467,8 @@ mod tests {
     fn jupyter_round_trip(contents: &str) -> String {
         let indentation = Indentation::default();
         let line_ending = LineEnding::default();
-        let parsed = ruff_python_parser::parse(contents, Mode::Ipython).unwrap();
+        let parsed =
+            ruff_python_parser::parse(contents, ParseOptions::from(Mode::Ipython)).unwrap();
         let Mod::Module(ModModule { body, .. }) = parsed.into_syntax() else {
             panic!("Source code didn't return ModModule")
         };

crates/ruff_python_codegen/src/stylist.rs

@@ -148,7 +148,7 @@ impl Deref for Indentation {
 
 #[cfg(test)]
 mod tests {
-    use ruff_python_parser::{parse_module, parse_unchecked, Mode};
+    use ruff_python_parser::{parse_module, parse_unchecked, Mode, ParseOptions};
     use ruff_source_file::{find_newline, LineEnding};
 
     use super::{Indentation, Quote, Stylist};
@@ -215,7 +215,7 @@ x = (
  3,
 )
 ";
-        let parsed = parse_unchecked(contents, Mode::Module);
+        let parsed = parse_unchecked(contents, ParseOptions::from(Mode::Module));
         assert_eq!(
             Stylist::from_tokens(parsed.tokens(), contents).indentation(),
             &Indentation(" ".to_string())

crates/ruff_python_formatter/src/cli.rs

@@ -7,7 +7,7 @@ use clap::{command, Parser, ValueEnum};
 
 use ruff_formatter::SourceCode;
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{parse, AsMode};
+use ruff_python_parser::{parse, ParseOptions};
 use ruff_python_trivia::CommentRanges;
 use ruff_text_size::Ranged;
 
@@ -48,7 +48,7 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re
     let source_type = PySourceType::from(source_path);
 
     // Parse the AST.
-    let parsed = parse(source, source_type.as_mode()).context("Syntax error in input")?;
+    let parsed = parse(source, ParseOptions::from(source_type)).context("Syntax error in input")?;
 
     let options = PyFormatOptions::from_extension(source_path)
         .with_preview(if cli.preview {

crates/ruff_python_formatter/src/comments/mod.rs

@@ -514,7 +514,7 @@ mod tests {
 
     use ruff_formatter::SourceCode;
     use ruff_python_ast::{Mod, PySourceType};
-    use ruff_python_parser::{parse, AsMode, Parsed};
+    use ruff_python_parser::{parse, ParseOptions, Parsed};
     use ruff_python_trivia::CommentRanges;
 
     use crate::comments::Comments;
@@ -529,8 +529,8 @@ mod tests {
         fn from_code(source: &'a str) -> Self {
             let source_code = SourceCode::new(source);
             let source_type = PySourceType::Python;
-            let parsed =
-                parse(source, source_type.as_mode()).expect("Expect source to be valid Python");
+            let parsed = parse(source, ParseOptions::from(source_type))
+                .expect("Expect source to be valid Python");
             let comment_ranges = CommentRanges::from(parsed.tokens());
 
             CommentsTestCase {

crates/ruff_python_formatter/src/lib.rs

@@ -5,7 +5,7 @@ pub use range::format_range;
 use ruff_formatter::prelude::*;
 use ruff_formatter::{format, write, FormatError, Formatted, PrintError, Printed, SourceCode};
 use ruff_python_ast::{AnyNodeRef, Mod};
-use ruff_python_parser::{parse, AsMode, ParseError, Parsed};
+use ruff_python_parser::{parse, ParseError, ParseOptions, Parsed};
 use ruff_python_trivia::CommentRanges;
 use ruff_text_size::Ranged;
 
@@ -112,7 +112,7 @@ pub fn format_module_source(
     options: PyFormatOptions,
 ) -> Result<Printed, FormatModuleError> {
     let source_type = options.source_type();
-    let parsed = parse(source, source_type.as_mode())?;
+    let parsed = parse(source, ParseOptions::from(source_type))?;
     let comment_ranges = CommentRanges::from(parsed.tokens());
     let formatted = format_module_ast(&parsed, &comment_ranges, source, options)?;
     Ok(formatted.print()?)
@@ -154,7 +154,7 @@ mod tests {
     use insta::assert_snapshot;
 
     use ruff_python_ast::PySourceType;
-    use ruff_python_parser::{parse, AsMode};
+    use ruff_python_parser::{parse, ParseOptions};
     use ruff_python_trivia::CommentRanges;
     use ruff_text_size::{TextRange, TextSize};
 
@@ -199,7 +199,7 @@ def main() -> None:
 
         // Parse the AST.
         let source_path = "code_inline.py";
-        let parsed = parse(source, source_type.as_mode()).unwrap();
+        let parsed = parse(source, ParseOptions::from(source_type)).unwrap();
         let comment_ranges = CommentRanges::from(parsed.tokens());
         let options = PyFormatOptions::from_extension(Path::new(source_path));
         let formatted = format_module_ast(&parsed, &comment_ranges, source, options).unwrap();

crates/ruff_python_formatter/src/range.rs

@@ -6,7 +6,7 @@ use ruff_formatter::{
 };
 use ruff_python_ast::visitor::source_order::{walk_body, SourceOrderVisitor, TraversalSignal};
 use ruff_python_ast::{AnyNodeRef, Stmt, StmtMatch, StmtTry};
-use ruff_python_parser::{parse, AsMode};
+use ruff_python_parser::{parse, ParseOptions};
 use ruff_python_trivia::{
     indentation_at_offset, BackwardsTokenizer, CommentRanges, SimpleToken, SimpleTokenKind,
 };
@@ -73,7 +73,7 @@ pub fn format_range(
 
     assert_valid_char_boundaries(range, source);
 
-    let parsed = parse(source, options.source_type().as_mode())?;
+    let parsed = parse(source, ParseOptions::from(options.source_type()))?;
     let source_code = SourceCode::new(source);
     let comment_ranges = CommentRanges::from(parsed.tokens());
     let comments = Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);

crates/ruff_python_formatter/src/string/docstring.rs

@@ -11,6 +11,7 @@ use regex::Regex;
 
 use ruff_formatter::printer::SourceMapGeneration;
 use ruff_python_ast::{str::Quote, AnyStringFlags, StringFlags};
+use ruff_python_parser::ParseOptions;
 use ruff_python_trivia::CommentRanges;
 use {
     ruff_formatter::{write, FormatOptions, IndentStyle, LineWidth, Printed},
@@ -492,8 +493,6 @@ impl<'src> DocstringLinePrinter<'_, '_, '_, 'src> {
         &mut self,
         kind: &mut CodeExampleKind<'_>,
     ) -> FormatResult<Option<Vec<OutputDocstringLine<'static>>>> {
-        use ruff_python_parser::AsMode;
-
         let line_width = match self.f.options().docstring_code_line_width() {
             DocstringCodeLineWidth::Fixed(width) => width,
             DocstringCodeLineWidth::Dynamic => {
@@ -570,7 +569,8 @@ impl<'src> DocstringLinePrinter<'_, '_, '_, 'src> {
                 std::format!(r#""""{}""""#, printed.as_code())
             }
         };
-        let result = ruff_python_parser::parse(&wrapped, self.f.options().source_type().as_mode());
+        let result =
+            ruff_python_parser::parse(&wrapped, ParseOptions::from(self.f.options().source_type()));
         // If the resulting code is not valid, then reset and pass through
         // the docstring lines as-is.
         if result.is_err() {
@@ -1580,10 +1580,8 @@ fn docstring_format_source(
     docstring_quote_style: Quote,
     source: &str,
 ) -> Result<Printed, FormatModuleError> {
-    use ruff_python_parser::AsMode;
-
     let source_type = options.source_type();
-    let parsed = ruff_python_parser::parse(source, source_type.as_mode())?;
+    let parsed = ruff_python_parser::parse(source, ParseOptions::from(source_type))?;
     let comment_ranges = CommentRanges::from(parsed.tokens());
     let source_code = ruff_formatter::SourceCode::new(source);
     let comments = crate::Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);

crates/ruff_python_formatter/tests/fixtures.rs

@@ -11,7 +11,7 @@ use crate::normalizer::Normalizer;
 use ruff_formatter::FormatOptions;
 use ruff_python_ast::comparable::ComparableMod;
 use ruff_python_formatter::{format_module_source, format_range, PreviewMode, PyFormatOptions};
-use ruff_python_parser::{parse, AsMode};
+use ruff_python_parser::{parse, ParseOptions};
 use ruff_source_file::{LineIndex, OneIndexed};
 use ruff_text_size::{TextRange, TextSize};
 
@@ -393,14 +393,14 @@ fn ensure_unchanged_ast(
     let source_type = options.source_type();
 
     // Parse the unformatted code.
-    let mut unformatted_ast = parse(unformatted_code, source_type.as_mode())
+    let mut unformatted_ast = parse(unformatted_code, ParseOptions::from(source_type))
         .expect("Unformatted code to be valid syntax")
         .into_syntax();
     Normalizer.visit_module(&mut unformatted_ast);
     let unformatted_ast = ComparableMod::from(&unformatted_ast);
 
     // Parse the formatted code.
-    let mut formatted_ast = parse(formatted_code, source_type.as_mode())
+    let mut formatted_ast = parse(formatted_code, ParseOptions::from(source_type))
         .expect("Formatted code to be valid syntax")
         .into_syntax();
     Normalizer.visit_module(&mut formatted_ast);