diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index f8acfb0f5ee52b..0ac13b256eb030 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -1168,6 +1168,7 @@ def get_schema_metadata() -> Optional[SchemaMetadata]: schema_fields = get_schema_fields_for_hive_column( hive_column_name=partition_key["Name"], hive_column_type=partition_key.get("Type", "unknown"), + description=partition_key.get("Comment"), default_nullable=False, ) assert schema_fields diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index a0a4ab00a429ea..7460e21e887ebe 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -113,6 +113,7 @@ { "fieldPath": "[version=2.0].[type=int].yr", "nullable": true, + "description": "test comment", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -204,6 +205,7 @@ { "fieldPath": "[version=2.0].[type=string].year", "nullable": true, + "description": "partition test comment", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 6a5f47ef67cf53..f0d506cd59f2ea 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -115,6 +115,7 @@ { "fieldPath": "[version=2.0].[type=int].yr", "nullable": true, + "description": "test comment", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -206,6 +207,7 @@ { "fieldPath": "[version=2.0].[type=string].year", "nullable": true, + "description": "partition test comment", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/test_glue_source_stubs.py index ecfe625e0860d1..771a0389c1e651 100644 --- a/metadata-ingestion/tests/unit/test_glue_source_stubs.py +++ b/metadata-ingestion/tests/unit/test_glue_source_stubs.py @@ -92,7 +92,7 @@ "Retention": 0, "StorageDescriptor": { "Columns": [ - {"Name": "yr", "Type": "int"}, + {"Name": "yr", "Type": "int", "Comment": "test comment"}, {"Name": "flightdate", "Type": "string"}, {"Name": "uniquecarrier", "Type": "string"}, {"Name": "airlineid", "Type": "int"}, @@ -129,7 +129,9 @@ }, "StoredAsSubDirectories": False, }, - "PartitionKeys": [{"Name": "year", "Type": "string"}], + "PartitionKeys": [ + {"Name": "year", "Type": "string", "Comment": "partition test comment"} + ], "TableType": "EXTERNAL_TABLE", "Parameters": { "CrawlerSchemaDeserializerVersion": "1.0",