From e7aab821c2aea3ab398b5576838e27688b0295f4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 30 May 2021 19:24:47 -0500 Subject: [PATCH 01/23] pants: new pack_metadata plugin Add pack_metadata target and a tailor rule to automatically add it in every dir with pack.yaml. --- pants-plugins/pack_metadata/BUILD | 1 + pants-plugins/pack_metadata/__init__.py | 0 pants-plugins/pack_metadata/register.py | 5 +++ pants-plugins/pack_metadata/tailor.py | 50 +++++++++++++++++++++ pants-plugins/pack_metadata/target_types.py | 29 ++++++++++++ pants.toml | 1 + 6 files changed, 86 insertions(+) create mode 100644 pants-plugins/pack_metadata/BUILD create mode 100644 pants-plugins/pack_metadata/__init__.py create mode 100644 pants-plugins/pack_metadata/register.py create mode 100644 pants-plugins/pack_metadata/tailor.py create mode 100644 pants-plugins/pack_metadata/target_types.py diff --git a/pants-plugins/pack_metadata/BUILD b/pants-plugins/pack_metadata/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/pack_metadata/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/pack_metadata/__init__.py b/pants-plugins/pack_metadata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py new file mode 100644 index 0000000000..4f9ce0b7f2 --- /dev/null +++ b/pants-plugins/pack_metadata/register.py @@ -0,0 +1,5 @@ +from pack_metadata.target_types import PackMetadata + + +def target_types(): + return [PackMetadata] diff --git a/pants-plugins/pack_metadata/tailor.py b/pants-plugins/pack_metadata/tailor.py new file mode 100644 index 0000000000..f22890be0d --- /dev/null +++ b/pants-plugins/pack_metadata/tailor.py @@ -0,0 +1,50 @@ +import os +from dataclasses import dataclass + +from pants.core.goals.tailor import ( + AllOwnedSources, + PutativeTarget, + PutativeTargets, + PutativeTargetsRequest, +) +from pants.engine.fs import PathGlobs, Paths +from pants.engine.rules import collect_rules, Get, rule, UnionRule +from pants.util.logging import LogLevel + +from pack_metadata.target_types import PackMetadata + + +@dataclass(frozen=True) +class PutativePackMetadataTargetsRequest: + pass + + +@rule( + desc="Find pack (config, action, alias, sensor, icon, etc) metadata files.", + level=LogLevel.DEBUG, +) +async def find_putative_targets( + _: PutativePackMetadataTargetsRequest, all_owned_sources: AllOwnedSources +) -> PutativeTargets: + all_pack_yaml_files = await Get(Paths, PathGlobs(["**/pack.yaml"])) + # pack_dirs = [os.path.dirname(p) for p in all_pack_yaml_files.files] + + unowned_pack_yaml_files = set(all_pack_yaml_files.files) - set(all_owned_sources) + unowned_pack_dirs = [os.path.dirname(p) for p in unowned_pack_yaml_files] + + name = "metadata" + return PutativeTargets( + [ + PutativeTarget.for_target_type( + PackMetadata, dirname, name, ("pack.yaml",), kwargs={"name": name} + ) + for dirname in unowned_pack_dirs + ] + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PutativeTargetsRequest, PutativePackMetadataTargetsRequest), + ] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py new file mode 100644 index 0000000000..a4eb27d1c9 --- /dev/null +++ b/pants-plugins/pack_metadata/target_types.py @@ -0,0 +1,29 @@ +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Target +from pants.core.target_types import FilesSources + + +class PackMetadataSources(FilesSources): + required = False + default = ( + # metadata does not include any python, shell, or other sources. + "pack.yaml", + "config.schema.yaml", + "*.yaml.example", + "**/*.yaml", + "**/*.yml", + "icon.png", # used in st2web ui + "requirements*.txt", + # "README.md", + # "HISTORY.md", + ) + + +class PackMetadata(Target): + alias = "pack_metadata" + core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSources) + help = ( + "Loose pack metadata files.\n\n" + "Pack metadata includes top-level files (pack.yaml, .yaml.examle, " + "config.schema.yaml, icon.png, and requirements.txt) and metadata for actions, " + "action-aliases, policies, rules, and sensors." + ) diff --git a/pants.toml b/pants.toml index 15a057cf02..375f9a1843 100644 --- a/pants.toml +++ b/pants.toml @@ -25,6 +25,7 @@ backend_packages = [ # internal plugins in pants-plugins/ "pants.backend.plugin_development", "api_spec", + "pack_metadata", "sample_conf", "schemas", ] From 77af55db3d621305d08df278ea4de07ad60bf91a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 30 May 2021 19:28:57 -0500 Subject: [PATCH 02/23] pants: register tailor rules in pack_metadata plugin --- pants-plugins/pack_metadata/register.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 4f9ce0b7f2..717a21d7b5 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -1,5 +1,10 @@ +from pack_metadata import tailor from pack_metadata.target_types import PackMetadata +def rules(): + return tailor.rules() + + def target_types(): return [PackMetadata] From 41a00b8cd7014f966d3db92df1d58eecb103dd07 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 30 May 2021 19:34:07 -0500 Subject: [PATCH 03/23] pants: do not include requirements.txt in metadata target --- pants-plugins/pack_metadata/target_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index a4eb27d1c9..67a8b94a13 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -12,7 +12,7 @@ class PackMetadataSources(FilesSources): "**/*.yaml", "**/*.yml", "icon.png", # used in st2web ui - "requirements*.txt", + # "requirements*.txt", # including this causes target conflicts # "README.md", # "HISTORY.md", ) From e70f9263e0b725cffbc4b8e799a7cab9777aba0f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 5 Jun 2021 17:53:21 -0500 Subject: [PATCH 04/23] use pack_metadata for st2tests.fixtures.packs.test_content_version_fixture --- st2tests/st2tests/fixtures/packs/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 407369573e..42fd75a920 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -2,7 +2,7 @@ # which is a git submodule. # The test_content_version* targets are dependencies of ./test_content_version_fixture -resources( +pack_metadata( name="test_content_version_metadata", sources=[ "test_content_version/pack.yaml", From 13e83789d9ad53b44654cbcb82c221fb8936b418 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 5 Jun 2021 17:54:55 -0500 Subject: [PATCH 05/23] Try to print instructions if git submodule not initiated --- pants-plugins/unmatched_globs/BUILD | 1 + pants-plugins/unmatched_globs/__init__.py | 0 pants-plugins/unmatched_globs/register.py | 10 +++ pants-plugins/unmatched_globs/target_types.py | 19 +++++ .../unmatched_globs/target_types_rules.py | 79 +++++++++++++++++++ pants.toml | 1 + st2tests/st2tests/fixtures/packs/BUILD | 6 ++ 7 files changed, 116 insertions(+) create mode 100644 pants-plugins/unmatched_globs/BUILD create mode 100644 pants-plugins/unmatched_globs/__init__.py create mode 100644 pants-plugins/unmatched_globs/register.py create mode 100644 pants-plugins/unmatched_globs/target_types.py create mode 100644 pants-plugins/unmatched_globs/target_types_rules.py diff --git a/pants-plugins/unmatched_globs/BUILD b/pants-plugins/unmatched_globs/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/unmatched_globs/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/unmatched_globs/__init__.py b/pants-plugins/unmatched_globs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/unmatched_globs/register.py b/pants-plugins/unmatched_globs/register.py new file mode 100644 index 0000000000..74ed117ec8 --- /dev/null +++ b/pants-plugins/unmatched_globs/register.py @@ -0,0 +1,10 @@ +from unmatched_globs import target_types_rules +from unmatched_globs.target_types import UnmatchedGlobsTarget + + +def rules(): + return target_types_rules.rules() + + +def target_types(): + return [UnmatchedGlobsTarget] diff --git a/pants-plugins/unmatched_globs/target_types.py b/pants-plugins/unmatched_globs/target_types.py new file mode 100644 index 0000000000..5c5b8430c0 --- /dev/null +++ b/pants-plugins/unmatched_globs/target_types.py @@ -0,0 +1,19 @@ +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, StringField, Target + + +class MessageOnErrorField(StringField): + alias = "message_on_error" + required = True + help = "The message to warn with when the dependency globs do not match." + + +# See `target_types_rules.py` for a dependency injection rule. +class UnmatchedGlobsDependencies(Dependencies): + required = True + + +class UnmatchedGlobsTarget(Target): + alias = "unmatched_globs" + core_fields = (*COMMON_TARGET_FIELDS, UnmatchedGlobsDependencies, MessageOnErrorField) + help = "Declare an error message to show when dependency globs do not match." + diff --git a/pants-plugins/unmatched_globs/target_types_rules.py b/pants-plugins/unmatched_globs/target_types_rules.py new file mode 100644 index 0000000000..ddd4df7daf --- /dev/null +++ b/pants-plugins/unmatched_globs/target_types_rules.py @@ -0,0 +1,79 @@ +from pants.backend.python.dependency_inference.rules import import_rules +from pants.engine.addresses import Address, Addresses +from pants.engine.fs import ( + GlobExpansionConjunction, + GlobMatchErrorBehavior, + PathGlobs, + Paths, +) +from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule +from pants.engine.target import ( + DependenciesRequest, + InjectDependenciesRequest, + InjectedDependencies, + Sources, + WrappedTarget, +) + +from unmatched_globs.target_types import UnmatchedGlobsDependencies, MessageOnErrorField + + +class InjectUnmatchedGlobsDependenciesRequest(InjectDependenciesRequest): + inject_for = UnmatchedGlobsDependencies + + +# rule will not be visible in list of running rules +@rule(desc="Check for dependencies' unmatched globs to provide helpful error message.") +async def helpful_message_for_git_submodule( + request: InjectUnmatchedGlobsDependenciesRequest, +) -> InjectedDependencies: + original_tgt: WrappedTarget = await Get( + WrappedTarget, Address, request.dependencies_field.address + ) + + dependencies = await Get( + Addresses, + DependenciesRequest(original_tgt.target[UnmatchedGlobsDependencies]) + ) + dependency_targets = await MultiGet( + Get(WrappedTarget, Address, address) + for address in dependencies + ) + + # the following is roughly based on Sources.path_globs + + sources_fields = [ + wrapped_tgt.target.get(field) + for wrapped_tgt in dependency_targets + for field in wrapped_tgt.target if (isinstance(field, Sources) and target.get(field)) + ] + # noinspection PyProtectedMember + source_globs = [ + source_field._prefix_glob_with_address(source_glob) + for source_field in sources_fields + if not source_field.default or set(source_field.value or ()) != set(source_field.default) + for source_glob in source_field.value or () + ] + + error_message = original_tgt.target[MessageOnErrorField] + + # Use the engine to warn when the globs do not match + await Get( + Paths, + PathGlobs( + source_globs, + conjunction=GlobExpansionConjunction.all_match, + glob_match_error_behavior=GlobMatchErrorBehavior.warn, + description_of_origin=error_message, + ) + ) + + return InjectedDependencies() + + +def rules(): + return [ + *collect_rules(), + *import_rules(), + UnionRule(InjectDependenciesRequest, InjectUnmatchedGlobsDependenciesRequest), + ] diff --git a/pants.toml b/pants.toml index 375f9a1843..bf1e47800b 100644 --- a/pants.toml +++ b/pants.toml @@ -28,6 +28,7 @@ backend_packages = [ "pack_metadata", "sample_conf", "schemas", + "unmatched_globs", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 42fd75a920..6f17a76194 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -2,6 +2,12 @@ # which is a git submodule. # The test_content_version* targets are dependencies of ./test_content_version_fixture +unmatched_globs( + name="git_submodule", + dependencies=[":test_content_version", ":test_content_version_metadata"], + message_on_error="You need git submodules! Please init ... instructions", +) + pack_metadata( name="test_content_version_metadata", sources=[ From 561079c67eef99c7a241b3055af45e86109b3575 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 7 Jun 2021 13:33:42 -0500 Subject: [PATCH 06/23] Try to use Sources.validate_resolved_files instead --- pants-plugins/unmatched_globs/register.py | 5 -- pants-plugins/unmatched_globs/target_types.py | 72 ++++++++++++++++- .../unmatched_globs/target_types_rules.py | 79 ------------------- 3 files changed, 69 insertions(+), 87 deletions(-) delete mode 100644 pants-plugins/unmatched_globs/target_types_rules.py diff --git a/pants-plugins/unmatched_globs/register.py b/pants-plugins/unmatched_globs/register.py index 74ed117ec8..0a5826ecad 100644 --- a/pants-plugins/unmatched_globs/register.py +++ b/pants-plugins/unmatched_globs/register.py @@ -1,10 +1,5 @@ -from unmatched_globs import target_types_rules from unmatched_globs.target_types import UnmatchedGlobsTarget -def rules(): - return target_types_rules.rules() - - def target_types(): return [UnmatchedGlobsTarget] diff --git a/pants-plugins/unmatched_globs/target_types.py b/pants-plugins/unmatched_globs/target_types.py index 5c5b8430c0..1407e68df4 100644 --- a/pants-plugins/unmatched_globs/target_types.py +++ b/pants-plugins/unmatched_globs/target_types.py @@ -1,4 +1,23 @@ -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, StringField, Target +import dataclasses + +from typing import Sequence + +from pants.engine.addresses import Address, Addresses +from pants.engine.fs import ( + GlobMatchErrorBehavior, + PathGlobs, + Paths, +) +from pants.engine.rules import Get, MultiGet +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + DependenciesRequest, + Sources, + StringField, + Target, + WrappedTarget, +) class MessageOnErrorField(StringField): @@ -12,8 +31,55 @@ class UnmatchedGlobsDependencies(Dependencies): required = True +class UnmatchedGlobsSources(Sources): + def validate_resolved_files(self, files: Sequence[str]) -> None: + original_tgt: WrappedTarget = await Get( + WrappedTarget, Address, self.address + ) + + error_message = original_tgt.target[MessageOnErrorField] + + dependencies = await Get( + Addresses, + DependenciesRequest(original_tgt.target[UnmatchedGlobsDependencies]) + ) + dependency_targets = await MultiGet( + Get(WrappedTarget, Address, address) + for address in dependencies + ) + + sources_fields = [ + wrapped_tgt.target.get(field) + for wrapped_tgt in dependency_targets + for field in wrapped_tgt.target + if (isinstance(field, Sources) and wrapped_tgt.target.get(field)) + ] + + path_globs = [ + dataclasses.replace( + source_field.path_globs(), + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=error_message, + ) + for source_field in sources_fields + ] + + # Use the engine to error when the globs do not match + await MultiGet( + Get( + Paths, + PathGlobs, + path_glob + ) for path_glob in path_globs + ) + + class UnmatchedGlobsTarget(Target): alias = "unmatched_globs" - core_fields = (*COMMON_TARGET_FIELDS, UnmatchedGlobsDependencies, MessageOnErrorField) + core_fields = ( + *COMMON_TARGET_FIELDS, + UnmatchedGlobsSources, + UnmatchedGlobsDependencies, + MessageOnErrorField, + ) help = "Declare an error message to show when dependency globs do not match." - diff --git a/pants-plugins/unmatched_globs/target_types_rules.py b/pants-plugins/unmatched_globs/target_types_rules.py deleted file mode 100644 index ddd4df7daf..0000000000 --- a/pants-plugins/unmatched_globs/target_types_rules.py +++ /dev/null @@ -1,79 +0,0 @@ -from pants.backend.python.dependency_inference.rules import import_rules -from pants.engine.addresses import Address, Addresses -from pants.engine.fs import ( - GlobExpansionConjunction, - GlobMatchErrorBehavior, - PathGlobs, - Paths, -) -from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule -from pants.engine.target import ( - DependenciesRequest, - InjectDependenciesRequest, - InjectedDependencies, - Sources, - WrappedTarget, -) - -from unmatched_globs.target_types import UnmatchedGlobsDependencies, MessageOnErrorField - - -class InjectUnmatchedGlobsDependenciesRequest(InjectDependenciesRequest): - inject_for = UnmatchedGlobsDependencies - - -# rule will not be visible in list of running rules -@rule(desc="Check for dependencies' unmatched globs to provide helpful error message.") -async def helpful_message_for_git_submodule( - request: InjectUnmatchedGlobsDependenciesRequest, -) -> InjectedDependencies: - original_tgt: WrappedTarget = await Get( - WrappedTarget, Address, request.dependencies_field.address - ) - - dependencies = await Get( - Addresses, - DependenciesRequest(original_tgt.target[UnmatchedGlobsDependencies]) - ) - dependency_targets = await MultiGet( - Get(WrappedTarget, Address, address) - for address in dependencies - ) - - # the following is roughly based on Sources.path_globs - - sources_fields = [ - wrapped_tgt.target.get(field) - for wrapped_tgt in dependency_targets - for field in wrapped_tgt.target if (isinstance(field, Sources) and target.get(field)) - ] - # noinspection PyProtectedMember - source_globs = [ - source_field._prefix_glob_with_address(source_glob) - for source_field in sources_fields - if not source_field.default or set(source_field.value or ()) != set(source_field.default) - for source_glob in source_field.value or () - ] - - error_message = original_tgt.target[MessageOnErrorField] - - # Use the engine to warn when the globs do not match - await Get( - Paths, - PathGlobs( - source_globs, - conjunction=GlobExpansionConjunction.all_match, - glob_match_error_behavior=GlobMatchErrorBehavior.warn, - description_of_origin=error_message, - ) - ) - - return InjectedDependencies() - - -def rules(): - return [ - *collect_rules(), - *import_rules(), - UnionRule(InjectDependenciesRequest, InjectUnmatchedGlobsDependenciesRequest), - ] From c6ef08c5150f29518fb6c87a5cc123af71ccbeb1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 7 Jun 2021 22:46:58 -0500 Subject: [PATCH 07/23] Try to use Sources.validate_resolved_files again --- pants-plugins/unmatched_globs/register.py | 7 +- pants-plugins/unmatched_globs/target_types.py | 129 ++++++++++-------- st2tests/st2tests/fixtures/packs/BUILD | 14 +- 3 files changed, 83 insertions(+), 67 deletions(-) diff --git a/pants-plugins/unmatched_globs/register.py b/pants-plugins/unmatched_globs/register.py index 0a5826ecad..6f9d857189 100644 --- a/pants-plugins/unmatched_globs/register.py +++ b/pants-plugins/unmatched_globs/register.py @@ -1,5 +1,8 @@ -from unmatched_globs.target_types import UnmatchedGlobsTarget +from unmatched_globs.target_types import ( + PackMetadataNoUnmatchedGlobs, + PythonLibraryNoUnmatchedGlobs, +) def target_types(): - return [UnmatchedGlobsTarget] + return [PackMetadataNoUnmatchedGlobs, PythonLibraryNoUnmatchedGlobs] diff --git a/pants-plugins/unmatched_globs/target_types.py b/pants-plugins/unmatched_globs/target_types.py index 1407e68df4..e37480f646 100644 --- a/pants-plugins/unmatched_globs/target_types.py +++ b/pants-plugins/unmatched_globs/target_types.py @@ -1,85 +1,98 @@ import dataclasses -from typing import Sequence +from typing import Any, Dict, Optional, Sequence +from typing_extensions import final -from pants.engine.addresses import Address, Addresses -from pants.engine.fs import ( - GlobMatchErrorBehavior, - PathGlobs, - Paths, +from pants.backend.python.target_types import ( + InterpreterConstraintsField, + PythonLibrary, + PythonLibrarySources, ) -from pants.engine.rules import Get, MultiGet +from pants.engine.addresses import Address from pants.engine.target import ( + AsyncFieldMixin, COMMON_TARGET_FIELDS, Dependencies, - DependenciesRequest, - Sources, StringField, Target, - WrappedTarget, ) +from pants.engine.unions import UnionMembership +from pack_metadata.target_types import PackMetadata, PackMetadataSources -class MessageOnErrorField(StringField): - alias = "message_on_error" - required = True - help = "The message to warn with when the dependency globs do not match." + +class UnmatchedGlobsError(Exception): + """Error thrown when a required set of globs didn't match.""" -# See `target_types_rules.py` for a dependency injection rule. -class UnmatchedGlobsDependencies(Dependencies): +class MessageOnUnmatchedGlobsField(StringField): + alias = "message_on_unmatched_globs" required = True + help = "The message to warn with when the sources field has unmatched globs." -class UnmatchedGlobsSources(Sources): +class NoUnmatchedGlobsSourcesMixin(AsyncFieldMixin): def validate_resolved_files(self, files: Sequence[str]) -> None: - original_tgt: WrappedTarget = await Get( - WrappedTarget, Address, self.address - ) + if not files: + raise UnmatchedGlobsError("inner unmatched globs error") - error_message = original_tgt.target[MessageOnErrorField] - dependencies = await Get( - Addresses, - DependenciesRequest(original_tgt.target[UnmatchedGlobsDependencies]) - ) - dependency_targets = await MultiGet( - Get(WrappedTarget, Address, address) - for address in dependencies - ) +class PythonLibrarySourcesNoUnmatchedGlobs( + NoUnmatchedGlobsSourcesMixin, PythonLibrarySources +): + required = True - sources_fields = [ - wrapped_tgt.target.get(field) - for wrapped_tgt in dependency_targets - for field in wrapped_tgt.target - if (isinstance(field, Sources) and wrapped_tgt.target.get(field)) - ] - - path_globs = [ - dataclasses.replace( - source_field.path_globs(), - glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin=error_message, - ) - for source_field in sources_fields - ] - - # Use the engine to error when the globs do not match - await MultiGet( - Get( - Paths, - PathGlobs, - path_glob - ) for path_glob in path_globs + +class PackMetadataSourcesNoUnmatchedGlobs( + NoUnmatchedGlobsSourcesMixin, PackMetadataSources +): + required = True + + +class UnmatchedGlobsTargetMixin(Target): + @final + def __init__( + self, + unhydrated_values: Dict[str, Any], + address: Address, + *, + union_membership: Optional[UnionMembership] = None, + ) -> None: + unmatched_globs_error_msg = unhydrated_values.get( + MessageOnUnmatchedGlobsField.alias, "There were unmatched_globs!" ) + try: + super(UnmatchedGlobsTargetMixin, self).__init__( + unhydrated_values, address, union_membership=union_membership + ) + except UnmatchedGlobsError: + raise UnmatchedGlobsError(unmatched_globs_error_msg) + + +_unmatched_globs_help = """ +The *_no_unmatched_globs variant errors if the sources field has unmatched globs. +When it errors, it prints the message from the `message_on_unmatched_globs` field. +""" + + +class PythonLibraryNoUnmatchedGlobs(UnmatchedGlobsTargetMixin, PythonLibrary): + alias = "python_library_no_unmatched_globs" + core_fields = ( + *COMMON_TARGET_FIELDS, + InterpreterConstraintsField, + Dependencies, + PythonLibrarySourcesNoUnmatchedGlobs, + MessageOnUnmatchedGlobsField, + ) + help = PythonLibrary.help + _unmatched_globs_help -class UnmatchedGlobsTarget(Target): - alias = "unmatched_globs" +class PackMetadataNoUnmatchedGlobs(UnmatchedGlobsTargetMixin, PackMetadata): + alias = "pack_metadata_no_unmatched_globs" core_fields = ( *COMMON_TARGET_FIELDS, - UnmatchedGlobsSources, - UnmatchedGlobsDependencies, - MessageOnErrorField, + Dependencies, + PackMetadataSourcesNoUnmatchedGlobs, + MessageOnUnmatchedGlobsField, ) - help = "Declare an error message to show when dependency globs do not match." + help = PackMetadata.help + _unmatched_globs_help diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 6f17a76194..bb63c4d293 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -2,13 +2,11 @@ # which is a git submodule. # The test_content_version* targets are dependencies of ./test_content_version_fixture -unmatched_globs( - name="git_submodule", - dependencies=[":test_content_version", ":test_content_version_metadata"], - message_on_error="You need git submodules! Please init ... instructions", -) +GIT_SUBMODULES_INSTRUCTIONS = """ +You need git submodules! Please init ... instructions +""" -pack_metadata( +pack_metadata_no_unmatched_globs( name="test_content_version_metadata", sources=[ "test_content_version/pack.yaml", @@ -16,6 +14,7 @@ pack_metadata( "test_content_version/icon.png", "test_content_version/requirements.txt", ], + message_on_unmatched_globs=GIT_SUBMODULES_INSTRUCTIONS, ) shell_sources( @@ -27,7 +26,7 @@ shell_sources( ], ) -python_sources( +python_sources_no_unmatched_globs( name="test_content_version", # do not fmt across git submodule boundary skip_black=True, @@ -38,4 +37,5 @@ python_sources( sources=[ "test_content_version/**/*.py", ], + message_on_unmatched_globs=GIT_SUBMODULES_INSTRUCTIONS, ) From 421231accc38788bcee469bf06cd2e5090776712 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 8 Jun 2021 10:56:08 -0500 Subject: [PATCH 08/23] Simplify pack_metadata_in_git_submodule We need to hard code the instructions in the plugin. So, we'll only add the instructions to the pack_metadata target. --- pants-plugins/unmatched_globs/register.py | 7 +- pants-plugins/unmatched_globs/target_types.py | 92 +++---------------- st2tests/st2tests/fixtures/packs/BUILD | 13 +-- 3 files changed, 18 insertions(+), 94 deletions(-) diff --git a/pants-plugins/unmatched_globs/register.py b/pants-plugins/unmatched_globs/register.py index 6f9d857189..b5603265d0 100644 --- a/pants-plugins/unmatched_globs/register.py +++ b/pants-plugins/unmatched_globs/register.py @@ -1,8 +1,5 @@ -from unmatched_globs.target_types import ( - PackMetadataNoUnmatchedGlobs, - PythonLibraryNoUnmatchedGlobs, -) +from unmatched_globs.target_types import PackMetadataInGitSubmodule def target_types(): - return [PackMetadataNoUnmatchedGlobs, PythonLibraryNoUnmatchedGlobs] + return [PackMetadataInGitSubmodule] diff --git a/pants-plugins/unmatched_globs/target_types.py b/pants-plugins/unmatched_globs/target_types.py index e37480f646..9707618e65 100644 --- a/pants-plugins/unmatched_globs/target_types.py +++ b/pants-plugins/unmatched_globs/target_types.py @@ -1,22 +1,5 @@ -import dataclasses - -from typing import Any, Dict, Optional, Sequence -from typing_extensions import final - -from pants.backend.python.target_types import ( - InterpreterConstraintsField, - PythonLibrary, - PythonLibrarySources, -) -from pants.engine.addresses import Address -from pants.engine.target import ( - AsyncFieldMixin, - COMMON_TARGET_FIELDS, - Dependencies, - StringField, - Target, -) -from pants.engine.unions import UnionMembership +from typing import Sequence +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies from pack_metadata.target_types import PackMetadata, PackMetadataSources @@ -25,74 +8,23 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" -class MessageOnUnmatchedGlobsField(StringField): - alias = "message_on_unmatched_globs" +class PackMetadataInGitSubmoduleSources(PackMetadataSources): required = True - help = "The message to warn with when the sources field has unmatched globs." - -class NoUnmatchedGlobsSourcesMixin(AsyncFieldMixin): def validate_resolved_files(self, files: Sequence[str]) -> None: if not files: - raise UnmatchedGlobsError("inner unmatched globs error") - - -class PythonLibrarySourcesNoUnmatchedGlobs( - NoUnmatchedGlobsSourcesMixin, PythonLibrarySources -): - required = True - - -class PackMetadataSourcesNoUnmatchedGlobs( - NoUnmatchedGlobsSourcesMixin, PackMetadataSources -): - required = True - - -class UnmatchedGlobsTargetMixin(Target): - @final - def __init__( - self, - unhydrated_values: Dict[str, Any], - address: Address, - *, - union_membership: Optional[UnionMembership] = None, - ) -> None: - unmatched_globs_error_msg = unhydrated_values.get( - MessageOnUnmatchedGlobsField.alias, "There were unmatched_globs!" - ) - try: - super(UnmatchedGlobsTargetMixin, self).__init__( - unhydrated_values, address, union_membership=union_membership - ) - except UnmatchedGlobsError: - raise UnmatchedGlobsError(unmatched_globs_error_msg) + raise UnmatchedGlobsError("Instructions go here") + super().validate_resolved_files(files) -_unmatched_globs_help = """ -The *_no_unmatched_globs variant errors if the sources field has unmatched globs. -When it errors, it prints the message from the `message_on_unmatched_globs` field. -""" - - -class PythonLibraryNoUnmatchedGlobs(UnmatchedGlobsTargetMixin, PythonLibrary): - alias = "python_library_no_unmatched_globs" - core_fields = ( - *COMMON_TARGET_FIELDS, - InterpreterConstraintsField, - Dependencies, - PythonLibrarySourcesNoUnmatchedGlobs, - MessageOnUnmatchedGlobsField, - ) - help = PythonLibrary.help + _unmatched_globs_help - - -class PackMetadataNoUnmatchedGlobs(UnmatchedGlobsTargetMixin, PackMetadata): - alias = "pack_metadata_no_unmatched_globs" +class PackMetadataInGitSubmodule(PackMetadata): + alias = "pack_metadata_in_git_submodule" core_fields = ( *COMMON_TARGET_FIELDS, Dependencies, - PackMetadataSourcesNoUnmatchedGlobs, - MessageOnUnmatchedGlobsField, + PackMetadataInGitSubmoduleSources, ) - help = PackMetadata.help + _unmatched_globs_help + help = PackMetadata.help + """ +The *_in_git_submodule variant errors if the sources field has unmatched globs. +It prints instructions on how to checkout the git submodules. +""" diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index bb63c4d293..c53022a3ca 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -1,12 +1,9 @@ # The files in test_content_version* targets are in ./test_content_version -# which is a git submodule. +# which is a git submodule. pack_metadata_in_git_submodule will error with +# instructions on how to checkout the submodules if they are not checked out. # The test_content_version* targets are dependencies of ./test_content_version_fixture -GIT_SUBMODULES_INSTRUCTIONS = """ -You need git submodules! Please init ... instructions -""" - -pack_metadata_no_unmatched_globs( +pack_metadata_in_git_submodule( name="test_content_version_metadata", sources=[ "test_content_version/pack.yaml", @@ -14,7 +11,6 @@ pack_metadata_no_unmatched_globs( "test_content_version/icon.png", "test_content_version/requirements.txt", ], - message_on_unmatched_globs=GIT_SUBMODULES_INSTRUCTIONS, ) shell_sources( @@ -26,7 +22,7 @@ shell_sources( ], ) -python_sources_no_unmatched_globs( +python_sources( name="test_content_version", # do not fmt across git submodule boundary skip_black=True, @@ -37,5 +33,4 @@ python_sources_no_unmatched_globs( sources=[ "test_content_version/**/*.py", ], - message_on_unmatched_globs=GIT_SUBMODULES_INSTRUCTIONS, ) From 94fc70405d0418a188b6febe7b4e59771357112e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 8 Jun 2021 11:02:40 -0500 Subject: [PATCH 09/23] move pack_metadata_in_git_submodule to pack_metadata plugin Since these instructions can't be generic enough to upstream, Move it to our pack_metadata plugin which is st2-specific. --- pants-plugins/pack_metadata/register.py | 4 +-- pants-plugins/pack_metadata/target_types.py | 29 ++++++++++++++++++ pants-plugins/unmatched_globs/BUILD | 1 - pants-plugins/unmatched_globs/__init__.py | 0 pants-plugins/unmatched_globs/register.py | 5 ---- pants-plugins/unmatched_globs/target_types.py | 30 ------------------- pants.toml | 1 - 7 files changed, 31 insertions(+), 39 deletions(-) delete mode 100644 pants-plugins/unmatched_globs/BUILD delete mode 100644 pants-plugins/unmatched_globs/__init__.py delete mode 100644 pants-plugins/unmatched_globs/register.py delete mode 100644 pants-plugins/unmatched_globs/target_types.py diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 717a21d7b5..a4166db992 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -1,5 +1,5 @@ from pack_metadata import tailor -from pack_metadata.target_types import PackMetadata +from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule def rules(): @@ -7,4 +7,4 @@ def rules(): def target_types(): - return [PackMetadata] + return [PackMetadata, PackMetadataInGitSubmodule] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 67a8b94a13..e5eecb403f 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -1,7 +1,13 @@ +from typing import Sequence + from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Target from pants.core.target_types import FilesSources +class UnmatchedGlobsError(Exception): + """Error thrown when a required set of globs didn't match.""" + + class PackMetadataSources(FilesSources): required = False default = ( @@ -18,6 +24,15 @@ class PackMetadataSources(FilesSources): ) +class PackMetadataInGitSubmoduleSources(PackMetadataSources): + required = True + + def validate_resolved_files(self, files: Sequence[str]) -> None: + if not files: + raise UnmatchedGlobsError("Instructions go here") + super().validate_resolved_files(files) + + class PackMetadata(Target): alias = "pack_metadata" core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSources) @@ -27,3 +42,17 @@ class PackMetadata(Target): "config.schema.yaml, icon.png, and requirements.txt) and metadata for actions, " "action-aliases, policies, rules, and sensors." ) + + +class PackMetadataInGitSubmodule(PackMetadata): + alias = "pack_metadata_in_git_submodule" + core_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + PackMetadataInGitSubmoduleSources, + ) + help = PackMetadata.help + ( + "\npack_metadata_in_git_submodule variant errors if the sources field " + "has unmatched globs. It prints instructions on how to checkout git " + "submodules." + ) diff --git a/pants-plugins/unmatched_globs/BUILD b/pants-plugins/unmatched_globs/BUILD deleted file mode 100644 index db46e8d6c9..0000000000 --- a/pants-plugins/unmatched_globs/BUILD +++ /dev/null @@ -1 +0,0 @@ -python_sources() diff --git a/pants-plugins/unmatched_globs/__init__.py b/pants-plugins/unmatched_globs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pants-plugins/unmatched_globs/register.py b/pants-plugins/unmatched_globs/register.py deleted file mode 100644 index b5603265d0..0000000000 --- a/pants-plugins/unmatched_globs/register.py +++ /dev/null @@ -1,5 +0,0 @@ -from unmatched_globs.target_types import PackMetadataInGitSubmodule - - -def target_types(): - return [PackMetadataInGitSubmodule] diff --git a/pants-plugins/unmatched_globs/target_types.py b/pants-plugins/unmatched_globs/target_types.py deleted file mode 100644 index 9707618e65..0000000000 --- a/pants-plugins/unmatched_globs/target_types.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import Sequence -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies - -from pack_metadata.target_types import PackMetadata, PackMetadataSources - - -class UnmatchedGlobsError(Exception): - """Error thrown when a required set of globs didn't match.""" - - -class PackMetadataInGitSubmoduleSources(PackMetadataSources): - required = True - - def validate_resolved_files(self, files: Sequence[str]) -> None: - if not files: - raise UnmatchedGlobsError("Instructions go here") - super().validate_resolved_files(files) - - -class PackMetadataInGitSubmodule(PackMetadata): - alias = "pack_metadata_in_git_submodule" - core_fields = ( - *COMMON_TARGET_FIELDS, - Dependencies, - PackMetadataInGitSubmoduleSources, - ) - help = PackMetadata.help + """ -The *_in_git_submodule variant errors if the sources field has unmatched globs. -It prints instructions on how to checkout the git submodules. -""" diff --git a/pants.toml b/pants.toml index bf1e47800b..375f9a1843 100644 --- a/pants.toml +++ b/pants.toml @@ -28,7 +28,6 @@ backend_packages = [ "pack_metadata", "sample_conf", "schemas", - "unmatched_globs", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ From 871618779762df0fb54927d135cd82311d6a5157 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 8 Jun 2021 14:03:15 -0500 Subject: [PATCH 10/23] Finish git submodule instructions --- pants-plugins/pack_metadata/target_types.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index e5eecb403f..1ca404bd4f 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -29,7 +29,12 @@ class PackMetadataInGitSubmoduleSources(PackMetadataSources): def validate_resolved_files(self, files: Sequence[str]) -> None: if not files: - raise UnmatchedGlobsError("Instructions go here") + raise UnmatchedGlobsError( + # see: st2tests.fixturesloader.GIT_SUBMODULES_NOT_CHECKED_OUT_ERROR + "One or more git submodules is not checked out. Make sure to run " + '"git submodule update --init --recursive"' + "in the repository root directory to check out all the submodules." + ) super().validate_resolved_files(files) From 835002d2e8107f71a888c9ac4c717f631903450c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 14 Jun 2021 12:48:29 -0500 Subject: [PATCH 11/23] add license header --- pants-plugins/pack_metadata/register.py | 13 +++++++++++++ pants-plugins/pack_metadata/tailor.py | 13 +++++++++++++ pants-plugins/pack_metadata/target_types.py | 13 +++++++++++++ 3 files changed, 39 insertions(+) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index a4166db992..34a6aa85cc 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -1,3 +1,16 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from pack_metadata import tailor from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule diff --git a/pants-plugins/pack_metadata/tailor.py b/pants-plugins/pack_metadata/tailor.py index f22890be0d..6a2b245fba 100644 --- a/pants-plugins/pack_metadata/tailor.py +++ b/pants-plugins/pack_metadata/tailor.py @@ -1,3 +1,16 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os from dataclasses import dataclass diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 1ca404bd4f..d798e64fbb 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -1,3 +1,16 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import Sequence from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Target From 892b0010f133dcf45b1c0aa62edd391dde39b121 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 26 May 2022 11:17:52 -0500 Subject: [PATCH 12/23] update to pants 2.8 (plugin apis, BUILD updates) --- pants-plugins/pack_metadata/tailor.py | 2 +- pants-plugins/pack_metadata/target_types.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pants-plugins/pack_metadata/tailor.py b/pants-plugins/pack_metadata/tailor.py index 6a2b245fba..452bae7eb6 100644 --- a/pants-plugins/pack_metadata/tailor.py +++ b/pants-plugins/pack_metadata/tailor.py @@ -28,7 +28,7 @@ @dataclass(frozen=True) -class PutativePackMetadataTargetsRequest: +class PutativePackMetadataTargetsRequest(PutativeTargetsRequest): pass diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index d798e64fbb..d6056fc941 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -14,14 +14,14 @@ from typing import Sequence from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Target -from pants.core.target_types import FilesSources +from pants.core.target_types import FilesGeneratingSourcesField class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" -class PackMetadataSources(FilesSources): +class PackMetadataSources(FilesGeneratingSourcesField): required = False default = ( # metadata does not include any python, shell, or other sources. From 84073a80b8e90393100ddcdd0176c885170f7ebe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Jun 2022 20:06:06 -0500 Subject: [PATCH 13/23] make api_spec and pack_metadata generate Resource targets --- pants-plugins/pack_metadata/target_types.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index d6056fc941..a4b3e7a43f 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -13,15 +13,18 @@ # limitations under the License. from typing import Sequence -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, Target -from pants.core.target_types import FilesGeneratingSourcesField +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies +from pants.core.target_types import ( + ResourcesGeneratingSourcesField, + ResourcesGeneratorTarget, +) class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" -class PackMetadataSources(FilesGeneratingSourcesField): +class PackMetadataSourcesField(ResourcesGeneratingSourcesField): required = False default = ( # metadata does not include any python, shell, or other sources. @@ -37,7 +40,7 @@ class PackMetadataSources(FilesGeneratingSourcesField): ) -class PackMetadataInGitSubmoduleSources(PackMetadataSources): +class PackMetadataInGitSubmoduleSources(PackMetadataSourcesField): required = True def validate_resolved_files(self, files: Sequence[str]) -> None: @@ -51,9 +54,9 @@ def validate_resolved_files(self, files: Sequence[str]) -> None: super().validate_resolved_files(files) -class PackMetadata(Target): +class PackMetadata(ResourcesGeneratorTarget): alias = "pack_metadata" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSources) + core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) help = ( "Loose pack metadata files.\n\n" "Pack metadata includes top-level files (pack.yaml, .yaml.examle, " From 4a1b241cb983ae468e265355d20849ae8a870b9c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 16 Jan 2023 20:09:24 -0600 Subject: [PATCH 14/23] Disable pack_metadata plugin until next PR --- pants.toml | 2 +- st2tests/st2tests/fixtures/packs/BUILD | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pants.toml b/pants.toml index 375f9a1843..e9bf57cb95 100644 --- a/pants.toml +++ b/pants.toml @@ -25,7 +25,7 @@ backend_packages = [ # internal plugins in pants-plugins/ "pants.backend.plugin_development", "api_spec", - "pack_metadata", + #"pack_metadata", "sample_conf", "schemas", ] diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index c53022a3ca..4813fd914d 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -3,7 +3,8 @@ # instructions on how to checkout the submodules if they are not checked out. # The test_content_version* targets are dependencies of ./test_content_version_fixture -pack_metadata_in_git_submodule( +# pack_metadata_in_git_submodule( +resources( name="test_content_version_metadata", sources=[ "test_content_version/pack.yaml", From 16213a1c0359278c3784e55a9626b7e3967eccb9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 16 Jan 2023 20:45:17 -0600 Subject: [PATCH 15/23] drop old comment --- pants-plugins/pack_metadata/tailor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pants-plugins/pack_metadata/tailor.py b/pants-plugins/pack_metadata/tailor.py index 452bae7eb6..88f714ce36 100644 --- a/pants-plugins/pack_metadata/tailor.py +++ b/pants-plugins/pack_metadata/tailor.py @@ -40,7 +40,6 @@ async def find_putative_targets( _: PutativePackMetadataTargetsRequest, all_owned_sources: AllOwnedSources ) -> PutativeTargets: all_pack_yaml_files = await Get(Paths, PathGlobs(["**/pack.yaml"])) - # pack_dirs = [os.path.dirname(p) for p in all_pack_yaml_files.files] unowned_pack_yaml_files = set(all_pack_yaml_files.files) - set(all_owned_sources) unowned_pack_dirs = [os.path.dirname(p) for p in unowned_pack_yaml_files] From b6074c4ebaef3a91da1108dcb9f46c594a904934 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 10:52:52 -0600 Subject: [PATCH 16/23] add tests for pants-plugins/pack_metadata --- pants-plugins/pack_metadata/BUILD | 4 + pants-plugins/pack_metadata/tailor_test.py | 107 ++++++++++++++++++ .../pack_metadata/target_types_test.py | 75 ++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 pants-plugins/pack_metadata/tailor_test.py create mode 100644 pants-plugins/pack_metadata/target_types_test.py diff --git a/pants-plugins/pack_metadata/BUILD b/pants-plugins/pack_metadata/BUILD index db46e8d6c9..0eea8b1cf1 100644 --- a/pants-plugins/pack_metadata/BUILD +++ b/pants-plugins/pack_metadata/BUILD @@ -1 +1,5 @@ python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/pack_metadata/tailor_test.py b/pants-plugins/pack_metadata/tailor_test.py new file mode 100644 index 0000000000..a6eea73c23 --- /dev/null +++ b/pants-plugins/pack_metadata/tailor_test.py @@ -0,0 +1,107 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.core.goals.tailor import ( + AllOwnedSources, + PutativeTarget, + PutativeTargets, +) +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .tailor import ( + PutativePackMetadataTargetsRequest, + rules as pack_metadata_rules, +) +from .target_types import PackMetadata, PackMetadataInGitSubmodule + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *pack_metadata_rules(), + QueryRule( + PutativeTargets, (PutativePackMetadataTargetsRequest, AllOwnedSources) + ), + ], + target_types=[PackMetadata, PackMetadataInGitSubmodule], + ) + + +def test_find_putative_targets(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "packs/already_owned/pack.yaml": "---\nname: already_owned\n", + "packs/already_owned/actions/action.yaml": "---\nname: action\n", + "packs/foo/pack.yaml": "---\nname: foo\n", + "packs/foo/actions/action.yaml": "---\nname: action\n", + "packs/bar/pack.yaml": "---\nname: bar\n", + "packs/bar/sensors/sensor.yaml": "---\nname: sensor\n", + "other/deep/baz/pack.yaml": "---\nname: baz\n", + } + ) + pts = rule_runner.request( + PutativeTargets, + [ + PutativePackMetadataTargetsRequest( + ( + "packs", + "packs/already_owned", + "packs/already_owned/actions", + "packs/foo", + "packs/foo/actions", + "packs/bar", + "packs/bar/sensors", + "other/deep/baz", + ) + ), + AllOwnedSources( + [ + "packs/already_owned/pack.yaml", + "packs/already_owned/actions/action.yaml", + ] + ), + ], + ) + assert ( + PutativeTargets( + [ + PutativeTarget.for_target_type( + PackMetadata, + path="packs/foo", + name="metadata", + triggering_sources=["packs/foo/pack.yaml"], + kwargs={"name": "metadata"}, + ), + PutativeTarget.for_target_type( + PackMetadata, + path="packs/bar", + name="metadata", + triggering_sources=["packs/bar/pack.yaml"], + kwargs={"name": "metadata"}, + ), + PutativeTarget.for_target_type( + PackMetadata, + path="other/deep/baz", + name="metadata", + triggering_sources=["other/deep/baz/pack.yaml"], + kwargs={"name": "metadata"}, + ), + ] + ) + == pts + ) diff --git a/pants-plugins/pack_metadata/target_types_test.py b/pants-plugins/pack_metadata/target_types_test.py new file mode 100644 index 0000000000..45279c5bae --- /dev/null +++ b/pants-plugins/pack_metadata/target_types_test.py @@ -0,0 +1,75 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.engine.addresses import Address +from pants.engine.target import SourcesPaths, SourcesPathsRequest +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .target_types import ( + PackMetadata, + # PackMetadataSourcesField, + PackMetadataInGitSubmodule, + PackMetadataInGitSubmoduleSources, + UnmatchedGlobsError, +) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + QueryRule(SourcesPaths, [SourcesPathsRequest]), + ], + target_types=[PackMetadata, PackMetadataInGitSubmodule], + ) + + +GIT_SUBMODULE_BUILD_FILE = """ +pack_metadata_in_git_submodule( + name="metadata", + sources=["./submodule_dir/pack.yaml"], +) +""" + + +def test_git_submodule_sources_missing(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "packs/BUILD": GIT_SUBMODULE_BUILD_FILE, + } + ) + tgt = rule_runner.get_target(Address("packs", target_name="metadata")) + + with pytest.raises(UnmatchedGlobsError): + _ = rule_runner.request( + SourcesPaths, [SourcesPathsRequest(tgt[PackMetadataInGitSubmoduleSources])] + ) + + +def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "packs/BUILD": GIT_SUBMODULE_BUILD_FILE, + "packs/submodule_dir/pack.yaml": "---\nname: foobar\n", + } + ) + tgt = rule_runner.get_target(Address("packs", target_name="metadata")) + + # basically: this asserts that it does not raise UnmatchedGlobsError + _ = rule_runner.request( + SourcesPaths, [SourcesPathsRequest(tgt[PackMetadataInGitSubmoduleSources])] + ) From 185e871260000b579902c9de449204f905959f2f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 11:14:25 -0600 Subject: [PATCH 17/23] add documentation for pants-plugins/pack_metadata --- pants-plugins/README.md | 23 +++++++++++++++++++++ pants-plugins/pack_metadata/target_types.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index d7b5729758..7f41eb2a26 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". +These StackStorm-specific plugins might be useful in other StackStorm-related repos. +- `pack_metadata` + These StackStorm-specific plugins are probably only useful for the st2 repo. - `api_spec` - `sample_conf` @@ -26,6 +29,26 @@ This plugin also wires up pants so that the `lint` goal runs additional api spec validation on `st2common/st2common/openapi.yaml` with something like `./pants lint st2common/st2common/openapi.yaml`. +### `pack_metadata` plugin + +This plugin adds two new targets to pants: +- `pack_metadata` +- `pack_metadata_in_git_submodule` + +These targets include all StackStorm pack metadata files in a pack. +Pack metadata includes top-level files (`pack.yaml`, `.yaml.examle`, +`config.schema.yaml`, and `icon.png`) and metadata (`*.yaml`, `*.yml`) +for actions, action-aliases, policies, rules, and sensors. + +This plugin also wires up the `tailor` goal, so that it will add a +`pack_metadata(name="metadata")` target wherever it finds a `pack.yaml` file. + +One of the packs in this repo is in a git submodule to test our handling +of git submodules (`st2tests/st2tests/fixtures/packs/test_content_version`). +If it is not checked out, then some of the tests will fail. +If it is not checked out, `pack_metadata_in_git_submodule` handles providing +a helpful, instructive error message as early as possible. + ### `sample_conf` plugin This plugin wires up pants to make sure `conf/st2.conf.sample` gets diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index a4b3e7a43f..667e531f91 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -60,7 +60,7 @@ class PackMetadata(ResourcesGeneratorTarget): help = ( "Loose pack metadata files.\n\n" "Pack metadata includes top-level files (pack.yaml, .yaml.examle, " - "config.schema.yaml, icon.png, and requirements.txt) and metadata for actions, " + "config.schema.yaml, and icon.png) and metadata for actions, " "action-aliases, policies, rules, and sensors." ) From 8e78e31366e1210fcaf291194c3baa64436e73c3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 12:37:46 -0600 Subject: [PATCH 18/23] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b19d3c0dce..728a95fb90 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 From 4d0bd8f50c05bbbd70f2263e250e87601be89c24 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 12:42:02 -0600 Subject: [PATCH 19/23] fix pants-plugins/pack_metadata test --- pants-plugins/pack_metadata/tailor_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pants-plugins/pack_metadata/tailor_test.py b/pants-plugins/pack_metadata/tailor_test.py index a6eea73c23..26974fbb83 100644 --- a/pants-plugins/pack_metadata/tailor_test.py +++ b/pants-plugins/pack_metadata/tailor_test.py @@ -84,21 +84,21 @@ def test_find_putative_targets(rule_runner: RuleRunner) -> None: PackMetadata, path="packs/foo", name="metadata", - triggering_sources=["packs/foo/pack.yaml"], + triggering_sources=["pack.yaml"], kwargs={"name": "metadata"}, ), PutativeTarget.for_target_type( PackMetadata, path="packs/bar", name="metadata", - triggering_sources=["packs/bar/pack.yaml"], + triggering_sources=["pack.yaml"], kwargs={"name": "metadata"}, ), PutativeTarget.for_target_type( PackMetadata, path="other/deep/baz", name="metadata", - triggering_sources=["other/deep/baz/pack.yaml"], + triggering_sources=["pack.yaml"], kwargs={"name": "metadata"}, ), ] From ac1c09783a19f0371d7a97a911ff103d42799ec4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 12:45:41 -0600 Subject: [PATCH 20/23] simplify and fix pants-plugins/pack_metadata test --- .../pack_metadata/target_types_test.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/pants-plugins/pack_metadata/target_types_test.py b/pants-plugins/pack_metadata/target_types_test.py index 45279c5bae..88bbfb4e85 100644 --- a/pants-plugins/pack_metadata/target_types_test.py +++ b/pants-plugins/pack_metadata/target_types_test.py @@ -16,14 +16,13 @@ import pytest from pants.engine.addresses import Address -from pants.engine.target import SourcesPaths, SourcesPathsRequest from pants.testutil.rule_runner import QueryRule, RuleRunner from .target_types import ( PackMetadata, # PackMetadataSourcesField, PackMetadataInGitSubmodule, - PackMetadataInGitSubmoduleSources, + # PackMetadataInGitSubmoduleSources, UnmatchedGlobsError, ) @@ -31,9 +30,7 @@ @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( - rules=[ - QueryRule(SourcesPaths, [SourcesPathsRequest]), - ], + rules=[], target_types=[PackMetadata, PackMetadataInGitSubmodule], ) @@ -52,12 +49,8 @@ def test_git_submodule_sources_missing(rule_runner: RuleRunner) -> None: "packs/BUILD": GIT_SUBMODULE_BUILD_FILE, } ) - tgt = rule_runner.get_target(Address("packs", target_name="metadata")) - with pytest.raises(UnmatchedGlobsError): - _ = rule_runner.request( - SourcesPaths, [SourcesPathsRequest(tgt[PackMetadataInGitSubmoduleSources])] - ) + tgt = rule_runner.get_target(Address("packs", target_name="metadata")) def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None: @@ -67,9 +60,5 @@ def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None: "packs/submodule_dir/pack.yaml": "---\nname: foobar\n", } ) - tgt = rule_runner.get_target(Address("packs", target_name="metadata")) - # basically: this asserts that it does not raise UnmatchedGlobsError - _ = rule_runner.request( - SourcesPaths, [SourcesPathsRequest(tgt[PackMetadataInGitSubmoduleSources])] - ) + tgt = rule_runner.get_target(Address("packs", target_name="metadata")) From 6e4afe062172ce573a49cd4416a7b8764882d419 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 12:54:36 -0600 Subject: [PATCH 21/23] fix exception check in pants-plugins/pack_metadata test --- pants-plugins/pack_metadata/target_types_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/target_types_test.py b/pants-plugins/pack_metadata/target_types_test.py index 88bbfb4e85..95d0ce40db 100644 --- a/pants-plugins/pack_metadata/target_types_test.py +++ b/pants-plugins/pack_metadata/target_types_test.py @@ -16,6 +16,7 @@ import pytest from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError from pants.testutil.rule_runner import QueryRule, RuleRunner from .target_types import ( @@ -49,8 +50,11 @@ def test_git_submodule_sources_missing(rule_runner: RuleRunner) -> None: "packs/BUILD": GIT_SUBMODULE_BUILD_FILE, } ) - with pytest.raises(UnmatchedGlobsError): - tgt = rule_runner.get_target(Address("packs", target_name="metadata")) + with pytest.raises(ExecutionError) as e: + _ = rule_runner.get_target(Address("packs", target_name="metadata")) + exc = e.value.wrapped_exceptions[0] + assert isinstance(exc, UnmatchedGlobsError) + assert "One or more git submodules is not checked out" in str(exc) def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None: From 6fbb1df20a2bfb821249c4be9d5d436d1a3163f5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 17 Jan 2023 13:00:56 -0600 Subject: [PATCH 22/23] flake8 fixes --- pants-plugins/pack_metadata/target_types_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/target_types_test.py b/pants-plugins/pack_metadata/target_types_test.py index 95d0ce40db..93a8a23292 100644 --- a/pants-plugins/pack_metadata/target_types_test.py +++ b/pants-plugins/pack_metadata/target_types_test.py @@ -17,7 +17,7 @@ from pants.engine.addresses import Address from pants.engine.internals.scheduler import ExecutionError -from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.testutil.rule_runner import RuleRunner from .target_types import ( PackMetadata, @@ -65,4 +65,4 @@ def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None: } ) # basically: this asserts that it does not raise UnmatchedGlobsError - tgt = rule_runner.get_target(Address("packs", target_name="metadata")) + _ = rule_runner.get_target(Address("packs", target_name="metadata")) From a73225c3a416c19a3dc1a0edf51ff48e2acf5202 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 20 Jan 2023 21:34:46 -0600 Subject: [PATCH 23/23] Fix typo Co-authored-by: Amanda McGuinness --- pants-plugins/README.md | 2 +- pants-plugins/pack_metadata/target_types.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index 7f41eb2a26..8f99aa3d5e 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -36,7 +36,7 @@ This plugin adds two new targets to pants: - `pack_metadata_in_git_submodule` These targets include all StackStorm pack metadata files in a pack. -Pack metadata includes top-level files (`pack.yaml`, `.yaml.examle`, +Pack metadata includes top-level files (`pack.yaml`, `.yaml.example`, `config.schema.yaml`, and `icon.png`) and metadata (`*.yaml`, `*.yml`) for actions, action-aliases, policies, rules, and sensors. diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 667e531f91..cfadbaab01 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -59,7 +59,7 @@ class PackMetadata(ResourcesGeneratorTarget): core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) help = ( "Loose pack metadata files.\n\n" - "Pack metadata includes top-level files (pack.yaml, .yaml.examle, " + "Pack metadata includes top-level files (pack.yaml, .yaml.example, " "config.schema.yaml, and icon.png) and metadata for actions, " "action-aliases, policies, rules, and sensors." )