Include data files in (backport) provider packages (apache#12200)

This isn't the most stable long-term solution but this works for now
where we've only got 4 dists the need this

.gitignore

@@ -189,6 +189,7 @@ log.txt*
 
 # Provider-related ignores
 /provider_packages/CHANGELOG.txt
+/provider_packages/MANIFEST.in
 /airflow/providers/__init__.py
 
 # Local .terraform directories

provider_packages/MANIFEST.in → dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2

@@ -16,6 +16,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
+{% if PROVIDER_PACKAGE_ID == 'amazon' %}
+include airflow/providers/amazon/aws/hooks/batch_waiters.json
+{% elif PROVIDER_PACKAGE_ID == 'cncf.kubernetes' %}
+include airflow/providers/cncf/kubernetes/example_dags/example_spark_kubernetes_spark_pi.yaml
+{% elif PROVIDER_PACKAGE_ID == 'google' %}
+include airflow/providers/google/cloud/example_dags/*.yaml
+include airflow/providers/google/cloud/example_dags/*.sql
+{% elif PROVIDER_PACKAGE_ID == 'papermill' %}
+include airflow/providers/papermill/example_dags/*.ipynb
+{% endif %}
+
 include NOTICE
 include LICENSE
 include CHANGELOG.txt

dev/provider_packages/SETUP_TEMPLATE.py.jinja2

@@ -59,7 +59,21 @@ def do_setup(version_suffix_for_pypi=''):
         packages=find_namespace_packages(
             include=['airflow.providers.{{ PROVIDER_PACKAGE_ID }}',
                      'airflow.providers.{{ PROVIDER_PACKAGE_ID }}.*']),
+        package_data={ '': [
+{% if PROVIDER_PACKAGE_ID == 'amazon' %}
+          "*.json",
+{% elif PROVIDER_PACKAGE_ID == 'cncf.kubernetes' %}
+          "*.yaml",
+{% elif PROVIDER_PACKAGE_ID == 'google' %}
+          "*.yaml",
+          "*.sql",
+{% elif PROVIDER_PACKAGE_ID == 'papermill' %}
+          "*.ipynb",
+{% endif %}
+            ],
+        },
         zip_safe=False,
+        include_package_data=True,
         install_requires={{ INSTALL_REQUIREMENTS }},
         setup_requires={{ SETUP_REQUIREMENTS }},
         extras_require={{ EXTRAS_REQUIREMENTS }},

dev/provider_packages/prepare_provider_packages.py

@@ -1220,6 +1220,7 @@ def update_generated_files_for_package(
     if update_setup:
         prepare_setup_py_file(context)
         prepare_setup_cfg_file(context)
+        prepare_manifest_in_file(context)
 
     bad = bad + sum([len(entity_summary.wrong_entities) for entity_summary in entity_summaries.values()])
     if bad != 0:
@@ -1318,6 +1319,19 @@ def prepare_setup_cfg_file(context):
         setup_file.write(setup_content)
 
 
+def prepare_manifest_in_file(context):
+    target = os.path.abspath(os.path.join(get_target_folder(), "MANIFEST.in"))
+    content = render_template(
+        template_name="MANIFEST",
+        context=context,
+        extension='.in',
+        autoescape=False,
+        keep_trailing_newline=True,
+    )
+    with open(target, "wt") as fh:
+        fh.write(content)
+
+
 def update_release_notes_for_packages(
     provider_ids: List[str], release_version: str, version_suffix: str, backport_packages: bool
 ):