Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# legacy deprecated ( do not use )
if 'path' in project.task: project.task['paths'] = [project.task['path']]
# loop all dt files to match pattern or match any pattern
print('PATHS', project.task['paths'])
for path in (project.task['paths'] or ['']):
print(path)
for dt_object in object_list(project.task['auth'], '%s:%s' % (project.task['bucket'], path), raw=True):
dt_size = dt_object['size']
dt_file = dt_object['name']
dt_time = dt_timestamp(dt_file)
dt_partition = dt_file.split('.', 1)[0]
if ((project.task.get('days') is None and project.task.get('hours') is None) or (dt_time > project.now - timedelta(days=project.task.get('days', 60), hours=project.task.get('hours', 0)))):
if not table_exists(project.task['to']['auth'], project.id, project.task['to']['dataset'], dt_partition):
dt_move(dt_object, dt_partition, jobs)
else:
if project.verbose: print ('DT Partition Exists:', dt_partition)
for count, job in enumerate(jobs):
print('Waiting For Job: %d of %d' % (count + 1, len(jobs)))
job_wait(project.task['to']['auth'], job)
@project.from_parameters
def dataset():
if project.verbose: print "DATASET", project.id, project.task['dataset']
# create dataset
datasets_create(project.task['auth'], project.id, project.task['dataset'])
datasets_access(project.task['auth'], project.id, project.task['dataset'], emails=project.task.get('emails', []), groups=project.task.get('groups', []))
def sheets():
if project.verbose: print('SHEETS')
# if sheet or tab is missing, don't do anything
if not project.task.get('sheet') or not project.task.get('tab'):
if project.verbose: print('Missing Sheet and/or Tab, skipping task.')
return
# delete if specified, will delete sheet if no more tabs remain
if project.task.get('delete', False):
sheets_tab_delete(
project.task['auth'],
project.task['sheet'],
project.task['tab']
)
# create a sheet and tab if specified, if template
if 'template' in project.task:
sheets_create(
project.task['auth'],
project.task['sheet'],
project.task['tab'],
project.task['template'].get('sheet'),
project.task['template'].get('tab'),
)
def ads():
"""Processes ads.
"""
placement_feed = Feed(project.task['auth'], project.task['sheet_id'],
'placement_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
event_tag_profile_feed = Feed(project.task['auth'], project.task['sheet_id'],
'event_tag_profile_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
ad_feed = Feed(project.task['auth'], project.task['sheet_id'], 'ad_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
ad_creative_assignment_feed = Feed(project.task['auth'],
project.task['sheet_id'],
'ad_creative_assignment_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
ad_placement_assignment_feed = Feed(project.task['auth'],
project.task['sheet_id'],
'ad_placement_assignment_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
ad_event_tag_assignment_feed = Feed(project.task['auth'],
project.task['sheet_id'],
'event_tag_ad_assignment_feed', spreadsheet=spreadsheet, timezone=project.task.get('timezone', None))
ad_dao.map_feeds(ad_feed.feed, ad_creative_assignment_feed.feed,
ad_placement_assignment_feed.feed,
ad_event_tag_assignment_feed.feed, placement_feed.feed,
event_tag_profile_feed.feed)
project.task['to']['view'],
query_parameters(project.task['from']['query'], project.task['from'].get('parameters')),
project.task['from'].get('legacy', project.task['from'].get('useLegacySql', True)), # DEPRECATED: useLegacySql
project.task['to'].get('replace', False)
)
else:
if project.verbose: print("STORAGE TO TABLE", project.task['to']['table'])
storage_to_table(
project.task['auth'],
project.id,
project.task['to']['dataset'],
project.task['to']['table'],
project.task['from']['bucket'] + ':' + project.task['from']['path'],
project.task.get('schema', []),
project.task.get('skip_rows', 1),
project.task.get('structure', 'CSV'),
project.task.get('disposition', 'WRITE_TRUNCATE')
)
schema = None
# RECOMMENDED: define schema in json
if project.task['out']['bigquery'].get('schema'):
if project.verbose: print('SHEETS SCHEMA DEFINED')
schema = project.task['out']['bigquery']['schema']
# NOT RECOMMENDED: determine schema if missing
else:
if project.verbose: print('SHEETS SCHEMA DETECT ( Note Recommended - Define Schema In JSON )')
# cast rows to types ( for schema detection )
rows = rows_to_type(rows)
rows, schema = get_schema(
rows,
project.task.get('header', False),
infer_type=project.task.get('infer_type', True)
)
# write to table ( not using put because no use cases for other destinations )
rows_to_table(
auth=project.task['out'].get('auth', project.task['auth']),
project_id=project.id,
dataset_id=project.task['out']['bigquery']['dataset'],
table_id=project.task['out']['bigquery']['table'],
rows=rows,
schema=schema,
skip_rows=1 if project.task.get('header', False) else 0,
disposition=project.task['out']['bigquery'].get('disposition', 'WRITE_TRUNCATE')
)
else:
print('SHEET EMPTY')
# project.task['template']['tab'],
# project.task['sheet'],
# project.task['tab'],
# project.task.get('clear', False)
# )
# if no template at least create tab
#else:
# sheets_tab_create(
# project.task['auth'],
# project.task['sheet'],
# project.task['tab']
# )
# clear if specified
if project.task.get('clear', False):
sheets_clear(
project.task['auth'],
project.task['sheet'],
project.task['tab'],
project.task.get('range', 'A1')
)
# write data if specified
if 'write' in project.task:
rows = get_rows(project.task['auth'], project.task['write'])
sheets_write(
project.task['auth'],
project.task['sheet'],
project.task['tab'],
project.task['range'],
rows,
# stores existing report json
report = None
# check if report is to be deleted
if project.task.get('delete', False):
if project.verbose: print('DCM DELETE', project.task['report'].get('name', None) or project.task['report'].get('body', {}).get('name', None) or project.task['report'].get('report_id', None))
report_delete(
project.task['auth'],
project.task['report']['account'],
project.task['report'].get('report_id', None),
project.task['report'].get('name', None) or project.task['report'].get('body', {}).get('name', None),
)
# check if report is to be run
if project.task.get('report_run_only', False):
if project.verbose: print('DCM REPORT RUN', project.task['report'].get('name', None) or project.task['report'].get('report_id', None))
report_run(
project.task['auth'],
project.task['report']['account'],
project.task['report'].get('report_id', None),
project.task['report'].get('name', None),
)
# check if report is to be created
if 'body' in project.task['report']:
if project.verbose: print('DCM BUILD', project.task['report']['body']['name'])
# filters can be passed using special get_rows handler, allows reading values from sheets etc...
if 'filters' in project.task['report']:
for f, d in project.task['report']['filters'].items():
for v in get_rows(project.task['auth'], d):
def ga_settings_download():
if project.verbose: print('Initiating GA Settings Download')
filters = [str(f) for f in project.task.get('accounts', [])] # in case integer list is provided
accounts = [a for a in API_Analytics(project.task['auth'], iterate=True).management().accountSummaries().list().execute() if not filters or a['id'] in filters or a['name'] in filters]
if accounts:
current_date = project.date
write_to_bigquery(
'ga_custom_dimension_settings',
CUSTOM_DIMENSION_SCHEMA,
custom_dimensions_download(accounts, current_date),
'CSV'
)
write_to_bigquery(
'ga_custom_metric_settings',
CUSTOM_METRIC_SCHEMA,
custom_metrics_download(accounts, current_date),