feat(FIR-36505): add where clause to copy from (#149)

firebolt-db · Sep 19, 2024 · c53bc05 · c53bc05
1 parent 8fa1211
commit c53bc05
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 4 deletions.
diff --git a/.changes/unreleased/Added-20240918-180154.yaml b/.changes/unreleased/Added-20240918-180154.yaml
@@ -0,0 +1,3 @@
+kind: Added
+body: Added ability to specify where clause in config for COPY FROM command
+time: 2024-09-18T18:01:54.377992+01:00
diff --git a/.github/workflows/jaffle_shop/sources_external_tables_copy.yml b/.github/workflows/jaffle_shop/sources_external_tables_copy.yml
@@ -25,4 +25,4 @@ sources:
               skip_blank_lines: true
               date_format: 'YYYY-MM-DD'
               timestamp_format: 'YYYY-MM-DD HH24:MI:SS'
-
+          where: "$source_file_timestamp > NOW() - interval '6 YEARS'"
diff --git a/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql b/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql
@@ -51,6 +51,7 @@
     {# COPY FROM is only available in Firebolt 2.0. #}
     {%- set external = source_node.external -%}
     {%- set credentials = external.credentials -%}
+    {%- set where_clause = external.where -%}
     {%- set options = external.options -%}
     {%- set csv_options = options.csv_options -%}
     {%- set error_file_credentials = options.error_file_credentials -%}
@@ -75,7 +76,7 @@
     )
     {%- endif %}
     FROM '{{external.url}}'
-    {%- if options %}
+    {%- if options or credentials %}
         WITH
         {%- if options.object_pattern %}
             PATTERN = '{{options.object_pattern}}'
@@ -130,8 +131,11 @@
                 TIMESTAMP_FORMAT = '{{ csv_options.timestamp_format }}'
             {%- endif %}
         {%- endif %}
+        {%- if credentials %}
+            CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
+        {%- endif %}
     {%- endif %}
-    {%- if credentials %}
-    CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
+    {%- if where_clause %}
+        WHERE {{ where_clause }}
     {%- endif %}
 {% endmacro %}