use pyupgrade to modernize syntax forward to 3.8 (#20317)

What, Python 3.8 you say? Hasn't Pants long been on 3.9? Yes, but 3.8 is
the last version for which pyupgrade supports
`--keep-runtime-typing`. Said runtime typing needs to be preserved for
now because Pants itself doesn't yet support the spiffy "new" typing
(`foo | None` instead of `Optional`).

build-support/bin/generate_docs.py

@@ -234,7 +234,7 @@ def run_pants_help_all() -> dict[str, Any]:
         "--no-verify-config",
         "help-all",
     ]
-    run = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
+    run = subprocess.run(argv, capture_output=True, encoding="utf-8")
     try:
         run.check_returncode()
     except subprocess.CalledProcessError:

pants.toml

@@ -207,7 +207,7 @@ template_by_globs = "@build-support/preambles/config.yaml"
 diff = true
 
 [pyupgrade]
-args = ["--keep-runtime-typing"]
+args = ["--keep-runtime-typing", "--py38-plus"]
 
 
 [jvm]

src/python/pants/backend/docker/goals/package_image.py

@@ -9,9 +9,7 @@
 from dataclasses import asdict, dataclass
 from functools import partial
 from itertools import chain
-from typing import Iterator, cast
-
-from typing_extensions import Literal
+from typing import Iterator, Literal, cast
 
 # Re-exporting BuiltDockerImage here, as it has its natural home here, but has moved out to resolve
 # a dependency cycle from docker_build_context.

src/python/pants/backend/go/util_rules/cgo.py

@@ -831,7 +831,7 @@ async def cgo_compile_request(
     # C files
     cflags = [*flags.cppflags, *flags.cflags]
     for gcc_file in gcc_files:
-        ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq))
+        ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o")
         oseq = oseq + 1
         out_obj_files.append(ofile)
 
@@ -850,7 +850,7 @@ async def cgo_compile_request(
     # C++ files
     cxxflags = [*flags.cppflags, *flags.cxxflags]
     for cxx_file in (os.path.join(dir_path, cxx_file) for cxx_file in request.cxx_files):
-        ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq))
+        ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o")
         oseq = oseq + 1
         out_obj_files.append(ofile)
 
@@ -868,7 +868,7 @@ async def cgo_compile_request(
 
     # Objective-C files
     for objc_file in (os.path.join(dir_path, objc_file) for objc_file in request.objc_files):
-        ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq))
+        ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o")
         oseq = oseq + 1
         out_obj_files.append(ofile)
 
@@ -888,7 +888,7 @@ async def cgo_compile_request(
     for fortran_file in (
         os.path.join(dir_path, fortran_file) for fortran_file in request.fortran_files
     ):
-        ofile = os.path.join(obj_dir_path, "_x{:03}.o".format(oseq))
+        ofile = os.path.join(obj_dir_path, f"_x{oseq:03}.o")
         oseq = oseq + 1
         out_obj_files.append(ofile)
 

src/python/pants/backend/go/util_rules/coverage_html.py

@@ -89,7 +89,7 @@ def _render_source_file(content: bytes, boundaries: Sequence[GoCoverageBoundary]
                 n = 0
                 if b.count > 0:
                     n = int(math.floor(b.norm * 9)) + 1
-                rendered.write('<span class="cov{}" title="{}">'.format(n, b.count))
+                rendered.write(f'<span class="cov{n}" title="{b.count}">')
             else:
                 rendered.write("</span>")
             boundaries = boundaries[1:]
@@ -154,7 +154,7 @@ async def render_go_coverage_profile_to_html(
                 {
                     "i": i,
                     "name": file.name,
-                    "coverage": "{:.1f}".format(file.coverage),
+                    "coverage": f"{file.coverage:.1f}",
                     "body": file.body,
                 }
                 for i, file in enumerate(files)

src/python/pants/backend/javascript/package_json.py

@@ -8,10 +8,9 @@
 import os.path
 from abc import ABC
 from dataclasses import dataclass, field
-from typing import Any, ClassVar, Iterable, Mapping, Optional, Tuple
+from typing import Any, ClassVar, Iterable, Literal, Mapping, Optional, Tuple
 
 import yaml
-from typing_extensions import Literal
 
 from pants.backend.project_info import dependencies
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior

src/python/pants/backend/project_info/peek.py

@@ -6,9 +6,7 @@
 import collections
 import json
 from dataclasses import dataclass, fields, is_dataclass
-from typing import Any, Iterable, Mapping
-
-from typing_extensions import Protocol, runtime_checkable
+from typing import Any, Iterable, Mapping, Protocol, runtime_checkable
 
 from pants.engine.addresses import Addresses
 from pants.engine.collection import Collection

src/python/pants/backend/python/framework/django/dependency_inference.py

@@ -86,7 +86,7 @@ async def _django_migration_dependencies(
     # default for decode(), we make that explicit here for emphasis.
     process_output = process_result.stdout.decode("utf8") or "{}"
     modules = [
-        "{}.migrations.{}".format(django_apps.label_to_name[label], migration)
+        f"{django_apps.label_to_name[label]}.migrations.{migration}"
         for label, migration in json.loads(process_output)
         if label in django_apps.label_to_name
     ]

src/python/pants/backend/python/util_rules/interpreter_constraints.py

@@ -6,10 +6,9 @@
 import itertools
 import re
 from collections import defaultdict
-from typing import Iterable, Iterator, Sequence, Tuple, TypeVar
+from typing import Iterable, Iterator, Protocol, Sequence, Tuple, TypeVar
 
 from pkg_resources import Requirement
-from typing_extensions import Protocol
 
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import InterpreterConstraintsField

src/python/pants/backend/python/util_rules/partition.py

@@ -5,9 +5,7 @@
 
 import itertools
 from collections import defaultdict
-from typing import Iterable, Mapping, Sequence, TypeVar
-
-from typing_extensions import Protocol
+from typing import Iterable, Mapping, Protocol, Sequence, TypeVar
 
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import InterpreterConstraintsField, PythonResolveField

src/python/pants/backend/python/util_rules/scripts/BUILD

@@ -1,3 +1,6 @@
 # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-python_sources()
+python_sources(
+    # Used for Python 2.7 distributions
+    skip_pyupgrade=True,
+)

src/python/pants/base/parse_context.py

@@ -2,9 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
-from typing import Any, Mapping
-
-from typing_extensions import Protocol
+from typing import Any, Mapping, Protocol
 
 
 class FilePathOracle(Protocol):

src/python/pants/base/specs.py

@@ -6,9 +6,7 @@
 import os
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import ClassVar, Iterable, Iterator, cast
-
-from typing_extensions import Protocol
+from typing import ClassVar, Iterable, Iterator, Protocol, cast
 
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.build_graph.address import Address

src/python/pants/bsp/protocol.py

@@ -4,7 +4,7 @@
 
 import logging
 from concurrent.futures import Future
-from typing import Any, BinaryIO, ClassVar
+from typing import Any, BinaryIO, ClassVar, Protocol
 
 from pylsp_jsonrpc.endpoint import Endpoint  # type: ignore[import]
 from pylsp_jsonrpc.exceptions import (  # type: ignore[import]
@@ -23,12 +23,6 @@
 from pants.engine.internals.selectors import Params
 from pants.engine.unions import UnionMembership, union
 
-try:
-    from typing import Protocol  # Python 3.8+
-except ImportError:
-    # See /~https://github.com/python/mypy/issues/4427 re the ignore
-    from typing_extensions import Protocol  # type: ignore
-
 _logger = logging.getLogger(__name__)
 
 

src/python/pants/core/goals/fix.py

@@ -7,9 +7,18 @@
 import logging
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Callable, Iterable, Iterator, NamedTuple, Sequence, Tuple, Type, TypeVar
-
-from typing_extensions import Protocol
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    Iterator,
+    NamedTuple,
+    Protocol,
+    Sequence,
+    Tuple,
+    Type,
+    TypeVar,
+)
 
 from pants.base.specs import Specs
 from pants.core.goals.lint import (

src/python/pants/core/goals/generate_lockfiles.py

@@ -8,9 +8,7 @@
 from collections import defaultdict
 from dataclasses import dataclass, replace
 from enum import Enum
-from typing import Callable, ClassVar, Iterable, Iterator, Mapping, Sequence, Tuple, cast
-
-from typing_extensions import Protocol
+from typing import Callable, ClassVar, Iterable, Iterator, Mapping, Protocol, Sequence, Tuple, cast
 
 from pants.engine.collection import Collection
 from pants.engine.console import Console

src/python/pants/core/goals/lint.py

@@ -6,9 +6,9 @@
 import logging
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Callable, ClassVar, Iterable, Iterator, Sequence, TypeVar, cast
+from typing import Any, Callable, ClassVar, Iterable, Iterator, Protocol, Sequence, TypeVar, cast
 
-from typing_extensions import Protocol, final
+from typing_extensions import final
 
 from pants.base.specs import Specs
 from pants.core.goals.multi_tool_goal_helper import (

src/python/pants/core/goals/multi_tool_goal_helper.py

@@ -5,9 +5,7 @@
 
 import logging
 import os.path
-from typing import Iterable, Mapping, Sequence, TypeVar
-
-from typing_extensions import Protocol
+from typing import Iterable, Mapping, Protocol, Sequence, TypeVar
 
 from pants.core.util_rules.distdir import DistDir
 from pants.engine.fs import EMPTY_DIGEST, Digest, Workspace

src/python/pants/core/util_rules/partitions.py

@@ -8,9 +8,7 @@
 import itertools
 from dataclasses import dataclass
 from enum import Enum
-from typing import Generic, Iterable, TypeVar, overload
-
-from typing_extensions import Protocol
+from typing import Generic, Iterable, Protocol, TypeVar, overload
 
 from pants.core.goals.multi_tool_goal_helper import SkippableSubsystem
 from pants.engine.collection import Collection

src/python/pants/engine/internals/dep_rules.py

@@ -6,8 +6,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from enum import Enum
-
-from typing_extensions import Protocol
+from typing import Protocol
 
 from pants.engine.addresses import Address
 from pants.engine.internals.target_adaptor import TargetAdaptor

src/python/pants/engine/internals/native_engine.pyi

@@ -12,14 +12,15 @@ from typing import (
     Generic,
     Iterable,
     Mapping,
+    Protocol,
     Sequence,
     TextIO,
     Tuple,
     TypeVar,
     overload,
 )
 
-from typing_extensions import Protocol, Self
+from typing_extensions import Self
 
 from pants.engine.internals.scheduler import Workunit, _PathGlobsAndRootCollection
 from pants.engine.internals.session import SessionValues

src/python/pants/engine/rules.py

@@ -16,6 +16,7 @@
     Iterable,
     Mapping,
     Optional,
+    Protocol,
     Sequence,
     Tuple,
     Type,
@@ -26,7 +27,7 @@
     overload,
 )
 
-from typing_extensions import ParamSpec, Protocol
+from typing_extensions import ParamSpec
 
 from pants.engine.engine_aware import SideEffecting
 from pants.engine.goal import Goal

src/python/pants/engine/target.py

@@ -29,6 +29,7 @@
     KeysView,
     Mapping,
     Optional,
+    Protocol,
     Sequence,
     Set,
     Tuple,
@@ -39,7 +40,7 @@
     get_type_hints,
 )
 
-from typing_extensions import Protocol, Self, final
+from typing_extensions import Self, final
 
 from pants.base.deprecated import warn_or_error
 from pants.engine.addresses import Address, Addresses, UnparsedAddressInputs, assert_single_address

src/python/pants/help/help_printer.py

@@ -6,9 +6,7 @@
 import re
 import textwrap
 from itertools import cycle
-from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, cast
-
-from typing_extensions import Literal
+from typing import Callable, Dict, Iterable, List, Literal, Optional, Set, Tuple, cast
 
 from pants.base.build_environment import pants_version
 from pants.help.help_formatter import HelpFormatter

src/python/pants/option/config.py

@@ -8,10 +8,9 @@
 import re
 from dataclasses import dataclass
 from types import SimpleNamespace
-from typing import Any, Dict, Iterable, List, Mapping, Union, cast
+from typing import Any, Dict, Iterable, List, Mapping, Protocol, Union, cast
 
 import toml
-from typing_extensions import Protocol
 
 from pants.base.build_environment import get_buildroot
 from pants.option.errors import ConfigError, ConfigValidationError, InterpolationMissingOptionError

src/python/pants/pantsd/pants_daemon_core.py

@@ -6,9 +6,7 @@
 import logging
 import threading
 from contextlib import contextmanager
-from typing import Iterator
-
-from typing_extensions import Protocol
+from typing import Iterator, Protocol
 
 from pants.build_graph.build_configuration import BuildConfiguration
 from pants.engine.env_vars import CompleteEnvironmentVars

src/python/pants/testutil/python_interpreter_selection.py

@@ -34,7 +34,7 @@ def has_python_version(version):
     return python_interpreter_path(version) is not None
 
 
-@lru_cache()
+@lru_cache
 def python_interpreter_path(version):
     """Returns the interpreter path if the current system has the specified version of python.
 

src/python/pants/testutil/skip_utils.py

@@ -7,7 +7,7 @@
 import pytest
 
 
-@lru_cache()
+@lru_cache
 def skip_if_command_errors(*args: str):
     def empty_decorator(func):
         return func

src/python/pants/util/dirutil.py

@@ -14,9 +14,7 @@
 from collections import defaultdict
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Callable, DefaultDict, Iterable, Iterator, Sequence, overload
-
-from typing_extensions import Literal
+from typing import Any, Callable, DefaultDict, Iterable, Iterator, Literal, Sequence, overload
 
 from pants.util.strutil import ensure_text