Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from typing import Dict, Optional
from airflow.operators.postgres_operator import PostgresOperator
from flowetl.mixins.table_name_macros_mixin import TableNameMacrosMixin
class CreateForeignStagingTableOperator(TableNameMacrosMixin, PostgresOperator):
def __init__(
self,
*,
filename: str,
fields: Dict[str, str],
program: Optional[str] = None,
header: bool = True,
delimiter: str = ",",
quote: str = '"',
escape: str = '"',
null: str = "",
encoding: Optional[str] = None,
**kwargs,
) -> None:
"""
Operator which uses file_fdw to create a table which can be used to read a flat file.
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from airflow.operators.sensors import SqlSensor
from airflow.utils.decorators import apply_defaults
from flowetl.mixins.table_name_macros_mixin import TableNameMacrosMixin
class FileFluxSensor(TableNameMacrosMixin, SqlSensor):
"""
The file flux sensor monitors a file for a short time to check if it is still
being modified.
Parameters
----------
conn_id : str
Connection to use
flux_check_interval : int
Number of seconds to wait between checks that a file is stable
filename : str
jinja templated string providing the path to the file to check
kwargs : dict
Passed to airflow.operators.sensors.SqlSensor
See Also
--------
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from airflow.operators.postgres_operator import PostgresOperator
from flowetl.mixins.table_name_macros_mixin import TableNameMacrosMixin
class AnalyzeOperator(TableNameMacrosMixin, PostgresOperator):
"""
The analyze operator triggers the postgres analyze command on a table.
Parameters
----------
target : str
jinja templated schema qualified table name.
kwargs : dict
Passed to airflow.operators.postgres_operator.PostgresOperator
"""
def __init__(self, *, target: str, **kwargs) -> None:
super().__init__(
sql=f"ANALYZE {target};", **kwargs
) # Need an f-string to let us use templating with the target