Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
]
expected = [
{'Name': 'locationid', 'Type': 'bigint'},
{'Name': 'borough', 'Type': 'string', 'Parameters': {'PII': 'PiiTypes.ADDRESS'}},
{'Name': 'zone', 'Type': 'string', 'Parameters': {'PII': 'PiiTypes.ADDRESS'}},
{'Name': 'service_zone', 'Type': 'string', 'Parameters': {'PII': 'PiiTypes.ADDRESS'}}
]
pii_table = {
'borough': ['PiiTypes.ADDRESS'],
'zone': ['PiiTypes.ADDRESS'],
'service_zone': ['PiiTypes.ADDRESS']
}
updated, is_updated = GlueStore.update_column_parameters(columns, pii_table)
self.assertTrue(is_updated)
self.assertEqual(expected, columns)
def test_param_no_update(self):
columns = [
{'Name': 'locationid', 'Type': 'bigint', 'Parameters': {'a': 'b'}}, {'Name': 'borough', 'Type': 'string'},
]
updated, is_updated = GlueStore.update_column_parameters(columns, {})
self.assertFalse(is_updated)
self.assertEqual(columns, updated)
def test_param_update(self):
columns = [
{'Name': 'locationid', 'Type': 'bigint', }, {'Name': 'borough', 'Type': 'string', 'Parameters': {'a': 'b'}},
]
pii_table = {
'borough': ['PiiTypes.ADDRESS'],
}
expected = [
{'Name': 'locationid', 'Type': 'bigint'},
{'Name': 'borough', 'Type': 'string', 'Parameters': {'a': 'b', 'PII': 'PiiTypes.ADDRESS'}}
]
updated, is_updated = GlueStore.update_column_parameters(columns, pii_table)
self.assertTrue(is_updated)
self.assertEqual(expected, columns)
'Name': 'dispatching_base_num', "Type": "string"
}, {
"Name": "pickup_datetime", "Type": "string"
}, {
"Name": "dropoff_datetime", "Type": "string"
}, {
"Name": "pulocationid", "Type": "bigint"
}, {
"Name": "dolocationid", "Type": "bigint"
}, {
"Name": "sr_flag", "Type": "bigint"
}, {
"Name": "hvfhs_license_num", "Type": "string"
}
]
updated, is_updated = GlueStore.update_column_parameters(columns, {})
self.assertFalse(is_updated)
self.assertEqual(columns, updated)
aws_access_key_id=explorer.config.access_key,
aws_secret_access_key=explorer.config.secret_key,
)
logging.debug(client)
for schema in schemas:
logging.debug("Processing schema {0}".format(schema.get_name()))
for table in schema.get_tables():
field_value = GlueStore.get_pii_table(table)
table_info = client.get_table(
DatabaseName=schema.get_name(), Name=table.get_name()
)
logging.debug(table_info)
updated_columns, is_table_updated = GlueStore.update_column_parameters(
table_info["Table"]["StorageDescriptor"]["Columns"], field_value
)
if is_table_updated:
updated_params = GlueStore.update_table_params(
table_info["Table"], updated_columns
)
client.update_table(
DatabaseName=schema.get_name(), TableInput=updated_params
)