diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 677af5dd79..9b7b381c1f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -65,7 +65,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 + #6254 #6258 #6259 #6260 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 59673bd746..9df7a372c9 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index de3b866405..ab10cd1c85 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/examples/actions/pythonactions/isprime.py b/contrib/examples/actions/pythonactions/isprime.py index 5116831a37..65294a5619 100644 --- a/contrib/examples/actions/pythonactions/isprime.py +++ b/contrib/examples/actions/pythonactions/isprime.py @@ -15,7 +15,8 @@ import math -from environ import get_environ +# TODO: extend pants and pants-plugins/pack_metadata to add lib dirs extra_sys_path for pylint +from environ import get_environ # pylint: disable=E0401 from st2common.runners.base_action import Action diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 8a73ff391a..201435eecc 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/pants-plugins/pack_metadata/python_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD new file mode 100644 index 0000000000..a172051977 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/BUILD @@ -0,0 +1,9 @@ +python_sources() + +python_tests( + name="tests", +) + +python_test_utils( + name="test_utils", +) diff --git a/pants-plugins/pack_metadata/python_rules/__init__.py b/pants-plugins/pack_metadata/python_rules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py new file mode 100644 index 0000000000..d481a1d5d1 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -0,0 +1,282 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from textwrap import dedent + +import pytest +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, +) +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.rules import QueryRule +from pants.testutil.python_rule_runner import PythonRuleRunner +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, + PytestPackTestRequest, +) +from pack_metadata.target_types import ( + InjectPackPythonPathField, + PackContentResourceTarget, + PackMetadata, +) + +# some random pack names +packs = ( + "foo", # imports between actions + "dr_seuss", # imports from /actions/lib + "shards", # imports from /lib + "metals", # imports the action from a subdirectory +) + + +@pytest.fixture +def pack_names() -> tuple[str, ...]: + return packs + + +def write_test_files(rule_runner: RuleRunner): + for pack in packs: + rule_runner.write_files( + { + f"packs/{pack}/BUILD": dedent( + """ + __defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata(name="metadata") + """ + ), + f"packs/{pack}/pack.yaml": dedent( + f""" + --- + name: {pack} + version: 1.0.0 + author: StackStorm + email: info@stackstorm.com + """ + ), + f"packs/{pack}/config.schema.yaml": "", + f"packs/{pack}/config.yaml.example": "", + f"packs/{pack}/icon.png": "", + f"packs/{pack}/README.md": f"# Pack {pack} README", + } + ) + + def action_metadata_file(action: str, entry_point: str = "") -> str: + entry_point = entry_point or f"{action}.py" + return dedent( + f""" + --- + name: {action} + runner_type: python-script + entry_point: {entry_point} + """ + ) + + def test_file(module: str, _object: str) -> str: + return dedent( + f""" + from {module} import {_object} + def test_{module.replace(".", "_")}() -> None: + pass + """ + ) + + rule_runner.write_files( + { + "packs/foo/actions/BUILD": "python_sources()", + "packs/foo/actions/get_bar.yaml": action_metadata_file("get_bar"), + "packs/foo/actions/get_bar.py": dedent( + """ + RESPONSE_CONSTANT = "foobar_key" + class BarAction: + def run(self): + return {RESPONSE_CONSTANT: "bar"} + """ + ), + "packs/foo/actions/get_baz.yaml": action_metadata_file("get_baz"), + "packs/foo/actions/get_baz.py": dedent( + """ + from get_bar import RESPONSE_CONSTANT + class BazAction: + def run(self): + return {RESPONSE_CONSTANT: "baz"} + """ + ), + "packs/foo/tests/BUILD": "python_tests()", + "packs/foo/tests/test_get_bar_action.py": test_file("get_bar", "BarAction"), + "packs/foo/tests/test_get_baz_action.py": test_file("get_baz", "BazAction"), + "packs/dr_seuss/actions/lib/seuss/BUILD": "python_sources()", + "packs/dr_seuss/actions/lib/seuss/__init__.py": "", + "packs/dr_seuss/actions/lib/seuss/things.py": dedent( + """ + THING1 = "thing one" + THING2 = "thing two" + """ + ), + "packs/dr_seuss/actions/BUILD": "python_sources()", + "packs/dr_seuss/actions/get_from_actions_lib.yaml": action_metadata_file( + "get_from_actions_lib" + ), + "packs/dr_seuss/actions/get_from_actions_lib.py": dedent( + """ + from seuss.things import THING1, THING2 + class GetFromActionsLibAction: + def run(self): + return {"things": (THING1, THING2)} + """ + ), + "packs/dr_seuss/tests/BUILD": "python_tests()", + "packs/dr_seuss/tests/test_get_from_actions_lib_action.py": test_file( + "get_from_actions_lib", "GetFromActionsLibAction" + ), + "packs/shards/lib/stormlight_archive/BUILD": "python_sources()", + "packs/shards/lib/stormlight_archive/__init__.py": "", + "packs/shards/lib/stormlight_archive/things.py": dedent( + """ + STORM_LIGHT = "Honor" + VOID_LIGHT = "Odium" + LIFE_LIGHT = "Cultivation" + """ + ), + "packs/shards/actions/BUILD": "python_sources()", + "packs/shards/actions/get_from_pack_lib.yaml": action_metadata_file( + "get_from_pack_lib" + ), + "packs/shards/actions/get_from_pack_lib.py": dedent( + """ + from stormlight_archive.things import STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT + class GetFromPackLibAction: + def run(self): + return {"light_sources": (STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT)} + """ + ), + "packs/shards/sensors/BUILD": "python_sources()", + "packs/shards/sensors/horn_eater.yaml": dedent( + """ + --- + name: horn_eater + entry_point: horn_eater.py + class_name: HornEaterSensor + trigger_types: [{name: horn_eater.saw.spren, payload_schema: {type: object}}] + """ + ), + "packs/shards/sensors/horn_eater.py": dedent( + """ + from st2reactor.sensor.base import PollingSensor + from stormlight_archive.things import STORM_LIGHT + class HornEaterSensor(PollingSensor): + def setup(self): pass + def poll(self): + if STORM_LIGHT in self.config: + self.sensor_service.dispatch( + trigger="horn_eater.saw.spren", payload={"spren_type": STORM_LIGHT} + ) + def cleanup(self): pass + def add_trigger(self): pass + def update_trigger(self): pass + def remove_trigger(self): pass + """ + ), + "packs/shards/tests/BUILD": "python_tests()", + "packs/shards/tests/test_get_from_pack_lib_action.py": test_file( + "get_from_pack_lib", "GetFromPackLibAction" + ), + "packs/shards/tests/test_horn_eater_sensor.py": test_file( + "horn_eater", "HornEaterSensor" + ), + "packs/metals/actions/fly.yaml": action_metadata_file( + "fly", "mist_born/fly.py" + ), + "packs/metals/actions/mist_born/BUILD": "python_sources()", + "packs/metals/actions/mist_born/__init__.py": "", + "packs/metals/actions/mist_born/fly.py": dedent( + """ + class FlyAction: + def run(self): + return {"metals": ("steel", "iron")} + """ + ), + "packs/metals/tests/BUILD": "python_tests()", + "packs/metals/tests/test_fly_action.py": test_file( + "mist_born.fly", "FlyAction" + ), + } + ) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = PythonRuleRunner( + rules=[ + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), + *python_target_types_rules(), + # TODO: not sure if we need a QueryRule for every rule... + *python_pack_content.rules(), + QueryRule( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest,), + ), + QueryRule( + PackContentPythonEntryPoints, (PackContentPythonEntryPointsRequest,) + ), + QueryRule(PackPythonLibs, (PackPythonLibsRequest,)), + *python_module_mapper.rules(), + QueryRule( + FirstPartyPythonMappingImpl, (St2PythonPackContentMappingMarker,) + ), + *python_path_rules.rules(), + QueryRule(PackPythonPath, (PackPythonPathRequest,)), + QueryRule(PytestPluginSetup, (PytestPackTestRequest,)), + ], + target_types=[ + PackContentResourceTarget, + PackMetadata, + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, + ], + ) + write_test_files(rule_runner) + args = ["--source-root-patterns=packs/*"] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + return rule_runner diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py new file mode 100644 index 0000000000..e35e93b997 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py @@ -0,0 +1,81 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from typing import DefaultDict + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + FirstPartyPythonMappingImplMarker, + ModuleProvider, + ModuleProviderType, + ResolveName, +) +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import PackMetadata + + +# This is only used to register our implementation with the plugin hook via unions. +class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): + pass + + +@rule( + desc=f"Creating map of `{PackMetadata.alias}` targets to Python modules in pack content", + level=LogLevel.DEBUG, +) +async def map_pack_content_to_python_modules( + _: St2PythonPackContentMappingMarker, +) -> FirstPartyPythonMappingImpl: + resolves_to_modules_to_providers: DefaultDict[ + ResolveName, DefaultDict[str, list[ModuleProvider]] + ] = defaultdict(lambda: defaultdict(list)) + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + for pack_content in pack_content_python_entry_points: + for module in pack_content.get_possible_modules(): + resolves_to_modules_to_providers[pack_content.resolve][module].append( + ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) + ) + + for pack_lib in pack_python_libs: + provider_type = ( + ModuleProviderType.TYPE_STUB + if pack_lib.relative_to_lib.suffix == ".pyi" + else ModuleProviderType.IMPL + ) + resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( + ModuleProvider(pack_lib.python_address, provider_type) + ) + + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) + + +def rules(): + return ( + *collect_rules(), + UnionRule(FirstPartyPythonMappingImplMarker, St2PythonPackContentMappingMarker), + ) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py new file mode 100644 index 0000000000..d1d6d779fb --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -0,0 +1,73 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + ModuleProvider, + ModuleProviderType, +) +from pants.engine.internals.native_engine import Address +from pants.testutil.rule_runner import RuleRunner +from pants.util.frozendict import FrozenDict + +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) + + +def test_map_pack_content_to_python_modules(rule_runner: RuleRunner) -> None: + result = rule_runner.request( + FirstPartyPythonMappingImpl, + (St2PythonPackContentMappingMarker(),), + ) + + def module_provider(spec_path: str, relative_file_path: str) -> ModuleProvider: + return ModuleProvider( + Address(spec_path=spec_path, relative_file_path=relative_file_path), + ModuleProviderType.IMPL, + ) + + expected = { + "": { + "get_bar": (module_provider("packs/foo/actions", "get_bar.py"),), + "get_baz": (module_provider("packs/foo/actions", "get_baz.py"),), + "seuss": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "__init__.py"), + ), + "seuss.things": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "things.py"), + ), + "get_from_actions_lib": ( + module_provider("packs/dr_seuss/actions", "get_from_actions_lib.py"), + ), + "stormlight_archive": ( + module_provider("packs/shards/lib/stormlight_archive", "__init__.py"), + ), + "stormlight_archive.things": ( + module_provider("packs/shards/lib/stormlight_archive", "things.py"), + ), + "get_from_pack_lib": ( + module_provider("packs/shards/actions", "get_from_pack_lib.py"), + ), + "horn_eater": (module_provider("packs/shards/sensors", "horn_eater.py"),), + "fly": (module_provider("packs/metals/actions/mist_born", "fly.py"),), + "mist_born.fly": ( + module_provider("packs/metals/actions/mist_born", "fly.py"), + ), + } + } + assert isinstance(result, FrozenDict) + assert all(isinstance(value, FrozenDict) for value in result.values()) + # pytest reports dict differences better than FrozenDict + assert {resolve: dict(value) for resolve, value in result.items()} == expected diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py new file mode 100644 index 0000000000..4fbe66ff69 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -0,0 +1,365 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import yaml +from collections import defaultdict +from dataclasses import dataclass +from pathlib import PurePath +from typing import DefaultDict + +from pants.backend.python.dependency_inference.module_mapper import ( + module_from_stripped_path, +) +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import PythonResolveField, PythonSourceField +from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior +from pants.base.specs import FileLiteralSpec, RawSpecs, RecursiveGlobSpec +from pants.engine.collection import Collection +from pants.engine.fs import DigestContents +from pants.engine.internals.native_engine import Address, Digest +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import ( + AllTargets, + AllUnexpandedTargets, + HydrateSourcesRequest, + HydratedSources, + Target, + Targets, +) +from pants.util.dirutil import fast_relpath +from pants.util.logging import LogLevel + +from pack_metadata.target_types import ( + PackContentResourceSourceField, + PackContentResourceTypeField, + PackContentResourceTypes, + PackMetadata, + PackMetadataSourcesField, +) + + +# Implementation Notes: +# +# With pants, we can rely on dependency inference for all the +# st2 components, runners, and other venv bits (st2 venv and pack venv). +# In StackStorm, all of that goes at the end of PYTHONPATH. +# Pants runs things hermetically via pex, so PYTHPNPATH +# changes happen via PEX_EXTRA_SYS_PATH instead. +# +# Actions: +# At runtime, the python_runner creates a PYTHONPATH that includes: +# [pack/lib:]pack_venv/lib/python3.x:pack_venv/lib/python3.x/site-packages:pack/actions/lib:st2_pythonpath +# python_runner runs python_action_wrapper which: +# - injects the action's entry_point's directory in sys.path +# - and then imports the action module and runs it. +# +# Sensors: +# At runtime, ProcessSensorContainer creates PYTHONPATH that includes: +# [pack/lib:]st2_pythonpath +# Then the process_container runs the sensor via sensor_wrapper which: +# - injects the sensor's entry_point's directory in sys.path +# (effectively always "sensors/" as a split("/") assumes only one dir) +# - and then imports the class_name from sensor module and runs it. +# +# For actions, this pants plugin should add this to PEX_EXTRA_SYS_PATH: +# pack/actions/path_to_entry_point:[pack/lib:]pack/actions/lib +# For sensors, this pants plugin should add this to PEX_EXTRA_SYS_PATH: +# pack/sensors:[pack/lib:] +# +# The rules in this file are used by: +# python_module_mapper.py: +# Dependency inference uses pack_metadata's module_mapper to detect any +# python imports that require one of these PYTHONPATH modifications, +# resolving those imports to modules in lib/, actions/, or sensors/. +# python_path_rules.py: +# Then get the relevant python imports from dependencies and +# add their parent directory to a generated PEX_EXTRA_SYS_PATH. + + +@dataclass(frozen=True) +class PackContentResourceTargetsOfTypeRequest: + types: tuple[PackContentResourceTypes, ...] + + +class PackContentResourceTargetsOfType(Targets): + pass + + +@rule( + desc=f"Find all `{PackMetadata.alias}` targets in project filtered by content type", + level=LogLevel.DEBUG, +) +async def find_pack_metadata_targets_of_types( + request: PackContentResourceTargetsOfTypeRequest, targets: AllTargets +) -> PackContentResourceTargetsOfType: + return PackContentResourceTargetsOfType( + tgt + for tgt in targets + if tgt.has_field(PackContentResourceSourceField) + and ( + not request.types + or tgt[PackContentResourceTypeField].value in request.types + ) + ) + + +@dataclass(frozen=True) +class PackContentPythonEntryPoint: + metadata_address: Address + content_type: PackContentResourceTypes + entry_point: str + python_address: Address + resolve: str + + @property + def python_file_path(self) -> PurePath: + return PurePath(self.python_address.filename) + + @staticmethod + def _split_pack_content_path(path: PurePath) -> tuple[PurePath, PurePath]: + content_types = ("actions", "sensors") # only content_types with python content + pack_content_dir = path.parent + while pack_content_dir.name not in content_types: + pack_content_dir = pack_content_dir.parent + relative_to_pack_content_dir = path.relative_to(pack_content_dir) + return pack_content_dir, relative_to_pack_content_dir + + def get_possible_modules(self) -> tuple[str, ...]: + """Get module names that could be imported. Mirrors get_possible_paths logic.""" + path = self.python_file_path + + # st2 adds the parent dir of the python file to sys.path at runtime. + module = path.stem if path.suffix == ".py" else path.name + modules = [module] + + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + _, relative_to_pack_content_dir = self._split_pack_content_path(path) + module = module_from_stripped_path(relative_to_pack_content_dir) + if module not in modules: + modules.append(module) + + return tuple(modules) + + def get_possible_paths(self) -> tuple[str, ...]: + """Get paths to add to PYTHONPATH and PEX_EXTRA_SYS_PATH. Mirrors get_possible_modules logic.""" + path = self.python_file_path + + # st2 adds the parent dir of the python file to sys.path at runtime. + paths = [path.parent.as_posix()] + + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + pack_content_dir, _ = self._split_pack_content_path(path) + if path.parent != pack_content_dir: + paths.append(pack_content_dir.as_posix()) + + return tuple(paths) + + +class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): + pass + + +class PackContentPythonEntryPointsRequest: + pass + + +@rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) +async def find_pack_content_python_entry_points( + python_setup: PythonSetup, _: PackContentPythonEntryPointsRequest +) -> PackContentPythonEntryPoints: + action_or_sensor = ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ) + + action_and_sensor_metadata_targets = await Get( + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest(action_or_sensor), + ) + action_and_sensor_metadata_sources = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(tgt[PackContentResourceSourceField])) + for tgt in action_and_sensor_metadata_targets + ) + action_and_sensor_metadata_contents = await MultiGet( + Get(DigestContents, Digest, source.snapshot.digest) + for source in action_and_sensor_metadata_sources + ) + + # python file path -> list of info about metadata files that refer to it + pack_content_entry_points_by_spec: DefaultDict[ + str, list[tuple[Address, PackContentResourceTypes, str]] + ] = defaultdict(list) + + tgt: Target + contents: DigestContents + for tgt, contents in zip( + action_and_sensor_metadata_targets, action_and_sensor_metadata_contents + ): + content_type = tgt[PackContentResourceTypeField].value + if content_type not in action_or_sensor: + continue + assert len(contents) == 1 + try: + metadata = yaml.safe_load(contents[0].content) or {} + except yaml.YAMLError: + continue + if content_type == PackContentResourceTypes.action_metadata: + runner_type = metadata.get("runner_type", "") or "" + if runner_type != "python-script": + # only python-script has special PYTHONPATH rules + continue + # get the entry_point to find subdirectory that contains the module + entry_point = metadata.get("entry_point", "") or "" + if entry_point: + # address.filename is basically f"{spec_path}/{relative_file_path}" + path = PurePath(tgt.address.filename).parent / entry_point + pack_content_entry_points_by_spec[str(path)].append( + (tgt.address, content_type, entry_point) + ) + + python_targets = await Get( + Targets, + RawSpecs( + file_literals=tuple( + FileLiteralSpec(spec_path) + for spec_path in pack_content_entry_points_by_spec + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata python module mapper", + ), + ) + + pack_content_entry_points: list[PackContentPythonEntryPoint] = [] + for tgt in python_targets: + if not tgt.has_field(PythonResolveField): + # this is unexpected + continue + for ( + metadata_address, + content_type, + entry_point, + ) in pack_content_entry_points_by_spec[tgt.address.filename]: + resolve = tgt[PythonResolveField].normalized_value(python_setup) + + pack_content_entry_points.append( + PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, + ) + ) + + return PackContentPythonEntryPoints(pack_content_entry_points) + + +@dataclass(frozen=True) +class PackPythonLib: + pack_path: PurePath + lib_dir: str + relative_to_lib: PurePath + python_address: Address + resolve: str + + @property + def module(self) -> str: + return module_from_stripped_path(self.relative_to_lib) + + @property + def lib_path(self) -> PurePath: + return self.pack_path / self.lib_dir + + +class PackPythonLibs(Collection[PackPythonLib]): + pass + + +class PackPythonLibsRequest: + pass + + +@rule(desc="Find all Pack lib directory python targets", level=LogLevel.DEBUG) +async def find_python_in_pack_lib_directories( + python_setup: PythonSetup, + all_unexpanded_targets: AllUnexpandedTargets, + _: PackPythonLibsRequest, +) -> PackPythonLibs: + pack_metadata_paths = [ + PurePath(tgt.address.spec_path) + for tgt in all_unexpanded_targets + if tgt.has_field(PackMetadataSourcesField) + ] + pack_lib_directory_targets = await MultiGet( + Get( + Targets, + RawSpecs( + recursive_globs=( + RecursiveGlobSpec(str(path / "lib")), + RecursiveGlobSpec(str(path / "actions" / "lib")), + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata lib directory lookup", + ), + ) + for path in pack_metadata_paths + ) + + # Maybe this should use this to take codegen into account. + # Get(PythonSourceFiles, PythonSourceFilesRequest(targets=lib_directory_targets, include_resources=False) + # For now, just take the targets as they are. + + pack_python_libs: list[PackPythonLib] = [] + + pack_path: PurePath + lib_directory_targets: Targets + for pack_path, lib_directory_targets in zip( + pack_metadata_paths, pack_lib_directory_targets + ): + for tgt in lib_directory_targets: + if not tgt.has_field(PythonSourceField): + # only python targets matter here. + continue + + relative_to_pack = PurePath( + fast_relpath(tgt[PythonSourceField].file_path, str(pack_path)) + ) + if relative_to_pack.parts[0] == "lib": + lib_dir = "lib" + elif relative_to_pack.parts[:2] == ("actions", "lib"): + lib_dir = "actions/lib" + else: + # This should not happen as it is not in the requested glob. + # Use this to tell linters that lib_dir is defined below here. + continue + relative_to_lib = relative_to_pack.relative_to(lib_dir) + + resolve = tgt[PythonResolveField].normalized_value(python_setup) + + pack_python_libs.append( + PackPythonLib( + pack_path=pack_path, + lib_dir=lib_dir, + relative_to_lib=relative_to_lib, + python_address=tgt.address, + resolve=resolve, + ) + ) + + return PackPythonLibs(pack_python_libs) + + +def rules(): + return (*collect_rules(),) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py new file mode 100644 index 0000000000..33c1389bb3 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -0,0 +1,158 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pants.engine.addresses import Address +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import PackContentResourceTypes + + +@pytest.mark.parametrize( + "requested_types,expected_count,expected_file_name", + ( + # one content type + ((PackContentResourceTypes.pack_metadata,), 4, "pack.yaml"), + ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), + ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), + ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), + ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), + ((PackContentResourceTypes.rule_metadata,), 0, ""), + ((PackContentResourceTypes.policy_metadata,), 0, ""), + ((PackContentResourceTypes.unknown,), 0, ""), + # all content types + ((), 22, ""), + # some content types + ( + ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ), + 6, + "", + ), + ( + ( + PackContentResourceTypes.pack_metadata, + PackContentResourceTypes.pack_config_schema, + PackContentResourceTypes.pack_config_example, + ), + 12, + "", + ), + ), +) +def test_find_pack_metadata_targets_of_types( + rule_runner: RuleRunner, + requested_types: tuple[PackContentResourceTypes, ...], + expected_count: int, + expected_file_name: str, +) -> None: + result = rule_runner.request( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest(requested_types),), + ) + assert len(result) == expected_count + if expected_file_name: + for tgt in result: + tgt.address.relative_file_path.endswith(expected_file_name) + + +def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: + result = rule_runner.request( + PackContentPythonEntryPoints, + (PackContentPythonEntryPointsRequest(),), + ) + assert len(result) == 6 # 5 actions + 1 sensor + assert {res.metadata_address for res in result} == { + Address( + "packs/foo", + relative_file_path="actions/get_bar.yaml", + target_name="metadata", + ), + Address( + "packs/foo", + relative_file_path="actions/get_baz.yaml", + target_name="metadata", + ), + Address( + "packs/dr_seuss", + relative_file_path="actions/get_from_actions_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="actions/get_from_pack_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="sensors/horn_eater.yaml", + target_name="metadata", + ), + Address( + "packs/metals", + relative_file_path="actions/fly.yaml", + target_name="metadata", + ), + } + assert {(res.content_type, res.entry_point) for res in result} == { + (PackContentResourceTypes.action_metadata, "get_bar.py"), + (PackContentResourceTypes.action_metadata, "get_baz.py"), + (PackContentResourceTypes.action_metadata, "get_from_actions_lib.py"), + (PackContentResourceTypes.action_metadata, "get_from_pack_lib.py"), + (PackContentResourceTypes.sensor_metadata, "horn_eater.py"), + (PackContentResourceTypes.action_metadata, "mist_born/fly.py"), + } + assert {res.python_address for res in result} == { + Address("packs/foo/actions", relative_file_path="get_bar.py"), + Address("packs/foo/actions", relative_file_path="get_baz.py"), + Address("packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py"), + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + } + + +def test_find_python_in_pack_lib_directories(rule_runner: RuleRunner) -> None: + result = rule_runner.request(PackPythonLibs, (PackPythonLibsRequest(),)) + assert len(result) == 4 + assert {(str(res.pack_path), res.lib_dir) for res in result} == { + ("packs/dr_seuss", "actions/lib"), + ("packs/shards", "lib"), + } + assert {res.python_address for res in result} == { + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py"), + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + Address("packs/shards/lib/stormlight_archive", relative_file_path="things.py"), + } + assert {res.module for res in result} == { + "seuss", + "seuss.things", + "stormlight_archive", + "stormlight_archive.things", + } diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py new file mode 100644 index 0000000000..314ecd7bf9 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -0,0 +1,131 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Set + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.engine.internals.native_engine import Address +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.target import Target, TransitiveTargets, TransitiveTargetsRequest +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel +from pants.util.ordered_set import OrderedSet + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import InjectPackPythonPathField + + +@dataclass(frozen=True) +class PackPythonPath: + entries: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PackPythonPathRequest: + address: Address + + +@rule( + desc="Get pack paths that should be added to PYTHONPATH/PEX_EXTRA_SYS_PATH for a target.", + level=LogLevel.DEBUG, +) +async def get_extra_sys_path_for_pack_dependencies( + request: PackPythonPathRequest, +) -> PackPythonPath: + transitive_targets = await Get( + TransitiveTargets, TransitiveTargetsRequest((request.address,)) + ) + + dependency_addresses: Set[Address] = { + tgt.address for tgt in transitive_targets.closure + } + if not dependency_addresses: + return PackPythonPath() + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + # only use addresses of actual dependencies + pack_python_content_addresses: Set[Address] = dependency_addresses & { + pack_content.python_address for pack_content in pack_content_python_entry_points + } + pack_python_lib_addresses: Set[Address] = dependency_addresses & { + pack_lib.python_address for pack_lib in pack_python_libs + } + + if not (pack_python_content_addresses or pack_python_lib_addresses): + return PackPythonPath() + + # filter pack_content_python_entry_points and pack_python_libs + pack_content_python_entry_points = ( + pack_content + for pack_content in pack_content_python_entry_points + if pack_content.python_address in pack_python_content_addresses + ) + pack_python_libs = ( + pack_lib + for pack_lib in pack_python_libs + if pack_lib.python_address in pack_python_lib_addresses + ) + + extra_sys_path_entries = OrderedSet() + for pack_content in pack_content_python_entry_points: + for path_entry in pack_content.get_possible_paths(): + extra_sys_path_entries.add(path_entry) + for pack_lib in pack_python_libs: + extra_sys_path_entries.add(pack_lib.lib_path.as_posix()) + + return PackPythonPath(tuple(extra_sys_path_entries)) + + +class PytestPackTestRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(InjectPackPythonPathField): + return False + return bool(target.get(InjectPackPythonPathField).value) + + +@rule( + desc="Inject pack paths in PYTHONPATH/PEX_EXTRA_SYS_PATH for python tests.", + level=LogLevel.DEBUG, +) +async def inject_extra_sys_path_for_pack_tests( + request: PytestPackTestRequest, +) -> PytestPluginSetup: + pack_python_path = await Get( + PackPythonPath, PackPythonPathRequest(request.target.address) + ) + return PytestPluginSetup( + # digest=EMPTY_DIGEST, + extra_sys_path=pack_python_path.entries, + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestPackTestRequest), + ] diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py new file mode 100644 index 0000000000..74ff010b40 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -0,0 +1,169 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.engine.internals.native_engine import Address, EMPTY_DIGEST +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, + PytestPackTestRequest, +) + + +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/actions", relative_file_path="get_bar.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/actions", relative_file_path="get_baz.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py" + ), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address( + "packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py" + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + ("packs/shards/lib",), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="things.py" + ), + ("packs/shards/lib",), + ), + ( + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="__init__.py"), + (), # there are no dependencies, and this is not an action entry point. + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_get_extra_sys_path_for_pack_dependencies( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + pack_python_path = rule_runner.request( + PackPythonPath, (PackPythonPathRequest(address),) + ) + assert pack_python_path.entries == expected + + +@pytest.mark.xfail(raises=AttributeError, reason="Not implemented in pants yet.") +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_inject_extra_sys_path_for_pack_tests( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + target = rule_runner.get_target(address) + result = rule_runner.request(PytestPluginSetup, (PytestPackTestRequest(target),)) + assert result.digest == EMPTY_DIGEST + assert result.extra_sys_path == expected diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 36c11079d9..6cdd7c9f8d 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,8 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, +) + from pack_metadata import tailor, target_types_rules +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) from pack_metadata.target_types import ( + InjectPackPythonPathField, + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, @@ -21,13 +34,21 @@ def rules(): return [ + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), *tailor.rules(), *target_types_rules.rules(), + *python_pack_content.rules(), + *python_module_mapper.rules(), + *python_path_rules.rules(), ] def target_types(): return [ + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 4c7c2c854f..01d80c24ad 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -11,12 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Sequence - -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies +from enum import Enum +from pathlib import PurePath +from typing import Optional, Sequence, Tuple + +from pants.engine.internals.native_engine import Address +from pants.engine.target import ( + BoolField, + COMMON_TARGET_FIELDS, + Dependencies, + StringField, +) from pants.core.target_types import ( + ResourceDependenciesField, ResourcesGeneratingSourcesField, ResourcesGeneratorTarget, + ResourcesOverridesField, + ResourceSourceField, + ResourceTarget, GenericTarget, ) @@ -25,6 +37,81 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" +class PackContentResourceTypes(Enum): + # in root of pack + pack_metadata = "pack_metadata" + pack_config_schema = "pack_config_schema" + pack_config_example = "pack_config_example" + pack_icon = "pack_icon" + # in subdirectory (see _content_type_by_path_parts below + action_metadata = "action_metadata" + action_chain_workflow = "action_chain_workflow" + orquesta_workflow = "orquesta_workflow" + alias_metadata = "alias_metadata" + policy_metadata = "policy_metadata" + rule_metadata = "rule_metadata" + sensor_metadata = "sensor_metadata" + trigger_metadata = "trigger_metadata" + # other + unknown = "unknown" + + +_content_type_by_path_parts: dict[Tuple[str, ...], PackContentResourceTypes] = { + ("actions",): PackContentResourceTypes.action_metadata, + ("actions", "chains"): PackContentResourceTypes.action_chain_workflow, + ("actions", "workflows"): PackContentResourceTypes.orquesta_workflow, + ("aliases",): PackContentResourceTypes.alias_metadata, + ("policies",): PackContentResourceTypes.policy_metadata, + ("rules",): PackContentResourceTypes.rule_metadata, + ("sensors",): PackContentResourceTypes.sensor_metadata, + ("triggers",): PackContentResourceTypes.trigger_metadata, +} + + +class PackContentResourceTypeField(StringField): + alias = "type" + help = ( + "The content type of the resource." + "\nDo not use this field in BUILD files. It is calculated automatically" + "based on the conventional location of files in the st2 pack." + ) + valid_choices = PackContentResourceTypes + value: PackContentResourceTypes + + @classmethod + def compute_value( + cls, raw_value: Optional[str], address: Address + ) -> PackContentResourceTypes: + value = super().compute_value(raw_value, address) + if value is not None: + return PackContentResourceTypes(value) + path = PurePath(address.relative_file_path) + _yaml_suffixes = (".yaml", ".yml") + if len(path.parent.parts) == 0: + # in the pack root + if path.stem == "pack" and path.suffix in _yaml_suffixes: + return PackContentResourceTypes.pack_metadata + if path.stem == "config.schema" and path.suffix in _yaml_suffixes: + return PackContentResourceTypes.pack_config_schema + if ( + path.stem.startswith("config.") + and path.suffixes[0] in _yaml_suffixes + and path.suffix == ".example" + ): + return PackContentResourceTypes.pack_config_example + if path.name == "icon.png": + return PackContentResourceTypes.pack_icon + return PackContentResourceTypes.unknown + resource_type = _content_type_by_path_parts.get(path.parent.parts, None) + if resource_type is not None: + return resource_type + return PackContentResourceTypes.unknown + + +class PackContentResourceSourceField(ResourceSourceField): + pass + + class PackMetadataSourcesField(ResourcesGeneratingSourcesField): required = False default = ( @@ -58,9 +145,26 @@ def validate_resolved_files(self, files: Sequence[str]) -> None: super().validate_resolved_files(files) +class PackContentResourceTarget(ResourceTarget): + alias = "pack_content_resource" + core_fields = ( + *COMMON_TARGET_FIELDS, + ResourceDependenciesField, + PackContentResourceSourceField, + PackContentResourceTypeField, + ) + help = "A single pack content resource file (mostly for metadata files)." + + class PackMetadata(ResourcesGeneratorTarget): alias = "pack_metadata" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) + core_fields = ( + *COMMON_TARGET_FIELDS, + PackMetadataSourcesField, + ResourcesOverridesField, + ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = ( "Loose pack metadata files.\n\n" "Pack metadata includes top-level files (pack.yaml, .yaml.example, " @@ -73,9 +177,11 @@ class PackMetadataInGitSubmodule(PackMetadata): alias = "pack_metadata_in_git_submodule" core_fields = ( *COMMON_TARGET_FIELDS, - Dependencies, PackMetadataInGitSubmoduleSources, + ResourcesOverridesField, ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = PackMetadata.help + ( "\npack_metadata_in_git_submodule variant errors if the sources field " "has unmatched globs. It prints instructions on how to checkout git " @@ -96,3 +202,14 @@ class PacksGlob(GenericTarget): "subdirectories (packs) except those listed with ! in dependencies. " "This is unfortunately needed by tests that use a glob to load pack fixtures." ) + + +class InjectPackPythonPathField(BoolField): + alias = "inject_pack_python_path" + help = ( + "For pack tests, set this to true to make sure /lib or actions/ dirs get " + "added to PYTHONPATH (actually PEX_EXTRA_SYS_PATH). Use `__defaults__` to enable " + "this in the BUILD file where you define pack_metadata, like this: " + "`__defaults__(all=dict(inject_pack_python_path=True))`" + ) + default = False diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 1b5b6e91a2..346f4ecf2e 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -28,7 +28,9 @@ def rules(): return [ - PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), + PythonTestsGeneratorTarget.register_plugin_field( + UsesServicesField, as_moved_field=True + ), PythonTestTarget.register_plugin_field(UsesServicesField), *platform_rules.rules(), *mongo_rules.rules(), diff --git a/pants.toml b/pants.toml index f532780afe..e4a673e2b8 100644 --- a/pants.toml +++ b/pants.toml @@ -90,9 +90,6 @@ root_patterns = [ "/contrib/packs", "/st2tests/testpacks/checks", "/st2tests/testpacks/errorcheck", - # pack common lib directories that ST2 adds to the PATH for actions/sensors - "/contrib/*/lib", - "/contrib/*/actions/lib", # other special-cased pack directories "/contrib/examples/actions/ubuntu_pkg_info", # python script runs via shell expecting cwd in PYTHONPATH # lint plugins