chore(ingest): reorganize unit tests (#11636)

datahub-project · Oct 17, 2024 · b814469 · b814469
1 parent d34717f
commit b814469
Show file tree

Hide file tree

Showing 42 changed files with 25 additions and 29 deletions.
diff --git a/metadata-ingestion/tests/unit/test_apis.py → ...ata-ingestion/tests/unit/api/test_apis.py b/metadata-ingestion/tests/unit/test_apis.py → ...ata-ingestion/tests/unit/api/test_apis.py
diff --git a/metadata-ingestion/tests/unit/test_report.py → ...sts/unit/api/test_entity_filter_report.py b/metadata-ingestion/tests/unit/test_report.py → ...sts/unit/api/test_entity_filter_report.py
diff --git a/...ata-ingestion/tests/unit/test_pipeline.py → ...ingestion/tests/unit/api/test_pipeline.py b/...ata-ingestion/tests/unit/test_pipeline.py → ...ingestion/tests/unit/api/test_pipeline.py
@@ -224,9 +224,9 @@ def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_pat
     def test_run_including_fake_transformation(self):
         pipeline = Pipeline.create(
             {
-                "source": {"type": "tests.unit.test_pipeline.FakeSource"},
+                "source": {"type": "tests.unit.api.test_pipeline.FakeSource"},
                 "transformers": [
-                    {"type": "tests.unit.test_pipeline.AddStatusRemovedTransformer"}
+                    {"type": "tests.unit.api.test_pipeline.AddStatusRemovedTransformer"}
                 ],
                 "sink": {"type": "tests.test_helpers.sink_helpers.RecordingSink"},
                 "run_id": "pipeline_test",
@@ -253,7 +253,7 @@ def test_run_including_registered_transformation(self):
 
         pipeline = Pipeline.create(
             {
-                "source": {"type": "tests.unit.test_pipeline.FakeSource"},
+                "source": {"type": "tests.unit.api.test_pipeline.FakeSource"},
                 "transformers": [
                     {
                         "type": "simple_add_dataset_ownership",
@@ -297,7 +297,7 @@ def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code
 ---
 run_id: pipeline_test
 source:
-    type: tests.unit.test_pipeline.{source}
+    type: tests.unit.api.test_pipeline.{source}
     config: {{}}
 sink:
     type: console
@@ -379,7 +379,7 @@ def test_pipeline_return_code(self, tmp_path, source, strict_warnings, exit_code
     def test_pipeline_process_commits(self, commit_policy, source, should_commit):
         pipeline = Pipeline.create(
             {
-                "source": {"type": f"tests.unit.test_pipeline.{source}"},
+                "source": {"type": f"tests.unit.api.test_pipeline.{source}"},
                 "sink": {"type": "console"},
                 "run_id": "pipeline_test",
             }

diff --git a/...ngestion/tests/unit/test_plugin_system.py → ...tion/tests/unit/api/test_plugin_system.py b/...ngestion/tests/unit/test_plugin_system.py → ...tion/tests/unit/api/test_plugin_system.py
diff --git a/...a-ingestion/tests/unit/api/test_report.py → ...tion/tests/unit/api/test_source_report.py b/...a-ingestion/tests/unit/api/test_report.py → ...tion/tests/unit/api/test_source_report.py
diff --git a/...ata-ingestion/tests/unit/test_workunit.py → ...ingestion/tests/unit/api/test_workunit.py b/...ata-ingestion/tests/unit/test_workunit.py → ...ingestion/tests/unit/api/test_workunit.py
diff --git a/...stion/tests/unit/test_bigquery_lineage.py → ...ts/unit/bigquery/test_bigquery_lineage.py b/...stion/tests/unit/test_bigquery_lineage.py → ...ts/unit/bigquery/test_bigquery_lineage.py
diff --git a/...tion/tests/unit/test_bigquery_profiler.py → ...s/unit/bigquery/test_bigquery_profiler.py b/...tion/tests/unit/test_bigquery_profiler.py → ...s/unit/bigquery/test_bigquery_profiler.py
diff --git a/...estion/tests/unit/test_bigquery_source.py → ...sts/unit/bigquery/test_bigquery_source.py b/...estion/tests/unit/test_bigquery_source.py → ...sts/unit/bigquery/test_bigquery_source.py
diff --git a/...n/tests/unit/test_bigquery_sql_lineage.py → ...nit/bigquery/test_bigquery_sql_lineage.py b/...n/tests/unit/test_bigquery_sql_lineage.py → ...nit/bigquery/test_bigquery_sql_lineage.py
diff --git a/...gestion/tests/unit/test_bigquery_usage.py → ...ests/unit/bigquery/test_bigquery_usage.py b/...gestion/tests/unit/test_bigquery_usage.py → ...ests/unit/bigquery/test_bigquery_usage.py
diff --git a/...ests/unit/test_bigqueryv2_usage_source.py → .../bigquery/test_bigqueryv2_usage_source.py b/...ests/unit/test_bigqueryv2_usage_source.py → .../bigquery/test_bigqueryv2_usage_source.py
diff --git a/...tests/unit/test_bq_get_partition_range.py → ...t/bigquery/test_bq_get_partition_range.py b/...tests/unit/test_bq_get_partition_range.py → ...t/bigquery/test_bq_get_partition_range.py
diff --git a/metadata-ingestion/tests/unit/test_check.py → ...ta-ingestion/tests/unit/cli/test_check.py b/metadata-ingestion/tests/unit/test_check.py → ...ta-ingestion/tests/unit/cli/test_check.py
diff --git a/...ngestion/tests/unit/test_check_upgrade.py → ...tion/tests/unit/cli/test_check_upgrade.py b/...ngestion/tests/unit/test_check_upgrade.py → ...tion/tests/unit/cli/test_check_upgrade.py
diff --git a/...ta-ingestion/tests/unit/test_cli_utils.py → ...ngestion/tests/unit/cli/test_cli_utils.py b/...ta-ingestion/tests/unit/test_cli_utils.py → ...ngestion/tests/unit/cli/test_cli_utils.py
diff --git a/...tion/tests/unit/test_key_value_pattern.py → ...sts/unit/config/test_key_value_pattern.py b/...tion/tests/unit/test_key_value_pattern.py → ...sts/unit/config/test_key_value_pattern.py
diff --git a/metadata-ingestion/tests/unit/glue/__init__.py b/metadata-ingestion/tests/unit/glue/__init__.py
diff --git a/...-ingestion/tests/unit/test_glue_source.py → ...stion/tests/unit/glue/test_glue_source.py b/...-ingestion/tests/unit/test_glue_source.py → ...stion/tests/unit/glue/test_glue_source.py
@@ -35,7 +35,7 @@
     validate_all_providers_have_committed_successfully,
 )
 from tests.test_helpers.type_helpers import PytestConfig
-from tests.unit.test_glue_source_stubs import (
+from tests.unit.glue.test_glue_source_stubs import (
     databases_1,
     databases_2,
     get_bucket_tagging,
@@ -71,6 +71,8 @@
 GMS_PORT = 8080
 GMS_SERVER = f"http://localhost:{GMS_PORT}"
 
+test_resources_dir = Path(__file__).parent
+
 
 def glue_source(
     platform_instance: Optional[str] = None,
@@ -247,7 +249,6 @@ def test_glue_ingest(
             write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
@@ -312,8 +313,6 @@ def test_config_without_platform():
 
 @freeze_time(FROZEN_TIME)
 def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
-
     deleted_actor_golden_mcs = "{}/glue_deleted_actor_mces_golden.json".format(
         test_resources_dir
     )
@@ -438,7 +437,6 @@ def test_glue_with_delta_schema_ingest(
         write_metadata_file(tmp_path / "glue_delta_mces.json", mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / "glue_delta_mces.json",
@@ -475,7 +473,6 @@ def test_glue_with_malformed_delta_schema_ingest(
         write_metadata_file(tmp_path / "glue_malformed_delta_mces.json", mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / "glue_malformed_delta_mces.json",
@@ -571,7 +568,6 @@ def test_glue_ingest_include_table_lineage(
             write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
@@ -678,7 +674,6 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass:
         write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,
@@ -716,7 +711,6 @@ def test_glue_ingest_with_profiling(
         write_metadata_file(tmp_path / mce_file, mce_objects)
 
     # Verify the output.
-    test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
     mce_helpers.check_golden_file(
         pytestconfig,
         output_path=tmp_path / mce_file,

diff --git a/...tion/tests/unit/test_glue_source_stubs.py → ...tests/unit/glue/test_glue_source_stubs.py b/...tion/tests/unit/test_glue_source_stubs.py → ...tests/unit/glue/test_glue_source_stubs.py
diff --git a/metadata-ingestion/tests/unit/redshift/__init__.py b/metadata-ingestion/tests/unit/redshift/__init__.py
diff --git a/...stion/tests/unit/redshift_query_mocker.py → ...ts/unit/redshift/redshift_query_mocker.py b/...stion/tests/unit/redshift_query_mocker.py → ...ts/unit/redshift/redshift_query_mocker.py
diff --git a/...estion/tests/unit/test_redshift_config.py → ...sts/unit/redshift/test_redshift_config.py b/...estion/tests/unit/test_redshift_config.py → ...sts/unit/redshift/test_redshift_config.py
diff --git a/...stion/tests/unit/test_redshift_lineage.py → ...ts/unit/redshift/test_redshift_lineage.py b/...stion/tests/unit/test_redshift_lineage.py → ...ts/unit/redshift/test_redshift_lineage.py
@@ -26,7 +26,7 @@
     SqlParsingDebugInfo,
     SqlParsingResult,
 )
-from tests.unit.redshift_query_mocker import mock_cursor
+from tests.unit.redshift.redshift_query_mocker import mock_cursor
 
 
 def test_get_sources_from_query():

diff --git a/...estion/tests/unit/test_redshift_source.py → ...sts/unit/redshift/test_redshift_source.py b/...estion/tests/unit/test_redshift_source.py → ...sts/unit/redshift/test_redshift_source.py
diff --git a/metadata-ingestion/tests/unit/sagemaker/__init__.py b/metadata-ingestion/tests/unit/sagemaker/__init__.py
diff --git a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py
@@ -14,7 +14,7 @@
     job_types,
 )
 from tests.test_helpers import mce_helpers
-from tests.unit.test_sagemaker_source_stubs import (
+from tests.unit.sagemaker.test_sagemaker_source_stubs import (
     describe_endpoint_response_1,
     describe_endpoint_response_2,
     describe_feature_group_response_1,

diff --git a/...tests/unit/test_sagemaker_source_stubs.py → .../sagemaker/test_sagemaker_source_stubs.py b/...tests/unit/test_sagemaker_source_stubs.py → .../sagemaker/test_sagemaker_source_stubs.py
diff --git a/...ingestion/tests/unit/graph/test_client.py → ...a-ingestion/tests/unit/sdk/test_client.py b/...ingestion/tests/unit/graph/test_client.py → ...a-ingestion/tests/unit/sdk/test_client.py
diff --git a/...ngestion/tests/unit/test_kafka_emitter.py → ...tion/tests/unit/sdk/test_kafka_emitter.py b/...ngestion/tests/unit/test_kafka_emitter.py → ...tion/tests/unit/sdk/test_kafka_emitter.py
diff --git a/...-ingestion/tests/unit/test_mce_builder.py → ...estion/tests/unit/sdk/test_mce_builder.py b/...-ingestion/tests/unit/test_mce_builder.py → ...estion/tests/unit/sdk/test_mce_builder.py
diff --git a/...-ingestion/tests/unit/test_mcp_builder.py → ...estion/tests/unit/sdk/test_mcp_builder.py b/...-ingestion/tests/unit/test_mcp_builder.py → ...estion/tests/unit/sdk/test_mcp_builder.py
diff --git a/...-ingestion/tests/unit/test_mcp_wrapper.py → ...estion/tests/unit/sdk/test_mcp_wrapper.py b/...-ingestion/tests/unit/test_mcp_wrapper.py → ...estion/tests/unit/sdk/test_mcp_wrapper.py
diff --git a/...ingestion/tests/unit/test_rest_emitter.py → ...stion/tests/unit/sdk/test_rest_emitter.py b/...ingestion/tests/unit/test_rest_emitter.py → ...stion/tests/unit/sdk/test_rest_emitter.py
diff --git a/...data-ingestion/tests/unit/test_codegen.py → ...ngestion/tests/unit/serde/test_codegen.py b/...data-ingestion/tests/unit/test_codegen.py → ...ngestion/tests/unit/serde/test_codegen.py
diff --git a/...a-ingestion/tests/unit/test_ldap_state.py → ...ateful_ingestion/state/test_ldap_state.py b/...a-ingestion/tests/unit/test_ldap_state.py → ...ateful_ingestion/state/test_ldap_state.py
diff --git a/...-ingestion/tests/unit/test_cli_logging.py → .../tests/unit/utilities/test_cli_logging.py b/...-ingestion/tests/unit/test_cli_logging.py → .../tests/unit/utilities/test_cli_logging.py
diff --git a/...-ingestion/tests/unit/test_ordered_set.py → .../tests/unit/utilities/test_ordered_set.py b/...-ingestion/tests/unit/test_ordered_set.py → .../tests/unit/utilities/test_ordered_set.py
diff --git a/metadata-ingestion/tests/unit/utilities/test_perf_timer.py b/metadata-ingestion/tests/unit/utilities/test_perf_timer.py
@@ -10,37 +10,39 @@
 
 def test_perf_timer_simple():
     with PerfTimer() as timer:
-        time.sleep(1)
-        assert approx(timer.elapsed_seconds()) == 1
+        time.sleep(0.4)
+        assert approx(timer.elapsed_seconds()) == 0.4
 
-    assert approx(timer.elapsed_seconds()) == 1
+    assert approx(timer.elapsed_seconds()) == 0.4
 
 
 def test_perf_timer_paused_timer():
     with PerfTimer() as current_timer:
-        time.sleep(1)
-        assert approx(current_timer.elapsed_seconds()) == 1
+        time.sleep(0.5)
+        assert approx(current_timer.elapsed_seconds()) == 0.5
         with current_timer.pause():
-            time.sleep(2)
-            assert approx(current_timer.elapsed_seconds()) == 1
-        assert approx(current_timer.elapsed_seconds()) == 1
-        time.sleep(1)
+            time.sleep(0.3)
+            assert approx(current_timer.elapsed_seconds()) == 0.5
+        assert approx(current_timer.elapsed_seconds()) == 0.5
+        time.sleep(0.2)
 
-    assert approx(current_timer.elapsed_seconds()) == 2
+    assert approx(current_timer.elapsed_seconds()) == 0.7
 
 
 def test_generator_with_paused_timer():
+    n = 4
+
     def generator_function():
         with PerfTimer() as inner_timer:
             time.sleep(1)
-            for i in range(10):
+            for i in range(n):
                 time.sleep(0.2)
                 with inner_timer.pause():
                     time.sleep(0.2)
                     yield i
-            assert approx(inner_timer.elapsed_seconds()) == 1 + 0.2 * 10
+            assert approx(inner_timer.elapsed_seconds()) == 1 + 0.2 * n
 
     with PerfTimer() as outer_timer:
         seq = generator_function()
         list([i for i in seq])
-        assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * 10 + 0.2 * 10
+        assert approx(outer_timer.elapsed_seconds()) == 1 + 0.2 * n + 0.2 * n
diff --git a/...n/tests/unit/test_serialized_lru_cache.py → ...it/utilities/test_serialized_lru_cache.py b/...n/tests/unit/test_serialized_lru_cache.py → ...it/utilities/test_serialized_lru_cache.py
diff --git a/...stion/tests/unit/test_topological_sort.py → ...s/unit/utilities/test_topological_sort.py b/...stion/tests/unit/test_topological_sort.py → ...s/unit/utilities/test_topological_sort.py
diff --git a/...ta-ingestion/tests/unit/test_utilities.py → ...on/tests/unit/utilities/test_utilities.py b/...ta-ingestion/tests/unit/test_utilities.py → ...on/tests/unit/utilities/test_utilities.py