feat(ingest/redshift): filter out system queries from usage (datahub-…

…project#10247)
sleeperdeep · Jun 25, 2024 · 689e3b8 · 689e3b8
1 parent 89b6e10
commit 689e3b8
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
@@ -678,6 +678,11 @@ def usage_query(start_time: str, end_time: str, database: str) -> str:
             AND ss.starttime < '{end_time}'
             AND sti.database = '{database}'
             AND sq.aborted = 0
+            AND NOT (
+                sq.querytxt LIKE 'small table validation: %'
+                OR sq.querytxt LIKE 'Small table conversion: %'
+                OR sq.querytxt LIKE 'padb_fetch_sample: %'
+            )
             ORDER BY ss.endtime DESC;
         """.strip()