Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
SITE_CONTACTS
)
if 'roles' in project.task['endpoints']:
# Roles
rows = get_roles(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Roles', ROLES_SCHEMA),
rows
)
if 'reports' in project.task['endpoints'] or project.task.get('reports') == True:
# Reports
rows = get_reports(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Reports', REPORTS_SCHEMA),
rows
)
# Reports Deliveries
if project.verbose: print('DCM Deliveries')
put_rows(
project.task['out']['auth'],
put_json('CM_Report_Deliveries', REPORT_DELIVERIES_SCHEMA),
REPORT_DELIVERIES
)
accounts = set(get_rows("user", project.task['accounts']))
if 'accounts' in project.task['endpoints']:
# Accounts
rows = get_accounts(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Accounts', ACCOUNTS_SCHEMA),
rows
)
if 'profiles' in project.task['endpoints']:
# Profiles
rows = get_profiles(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Profiles', PROFILES_SCHEMA),
rows
)
# Profiles Campaigns
if project.verbose: print('DCM Profile Campaigns')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Campaigns', PROFILE_CAMPAIGNS_SCHEMA),
PROFILE_CAMPAIGNS
)
# Profiles Sites
if project.verbose: print('DCM Profile Sites')
put_rows(
project.task['out']['auth'],
put_json('CM_Profiles', PROFILES_SCHEMA),
rows
)
# Profiles Campaigns
if project.verbose: print('DCM Profile Campaigns')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Campaigns', PROFILE_CAMPAIGNS_SCHEMA),
PROFILE_CAMPAIGNS
)
# Profiles Sites
if project.verbose: print('DCM Profile Sites')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Sites', PROFILE_SITES_SCHEMA),
PROFILE_SITES
)
# Profiles Roles
if project.verbose: print('DCM Profile Roles')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Roles', PROFILE_ROLES_SCHEMA),
PROFILE_ROLES
)
# Profiles Advertisers
if project.verbose: print('DCM Profile Advertisers')
put_rows(
if 'trends' in project.task:
if 'places' in project.task['trends']:
rows = twitter_trends_places()
project.task['out']['bigquery']['schema'] = TWITTER_TRENDS_PLACE_SCHEMA
project.task['out']['bigquery']['skip_rows'] = 0
elif 'closest' in project.task['trends']:
rows = twitter_trends_closest()
project.task['out']['bigquery']['schema'] = TWITTER_TRENDS_CLOSEST_SCHEMA
project.task['out']['bigquery']['skip_rows'] = 0
else:
rows = twitter_trends_available()
project.task['out']['bigquery']['schema'] = TWITTER_TRENDS_AVAILABLE_SCHEMA
project.task['out']['bigquery']['skip_rows'] = 0
if rows: put_rows(project.task['auth'], project.task['out'], 'twitter_%s.csv' % project.date, rows)
project.task['read']['out']['bigquery']['schema'] = schema
project.task['read']['out']['bigquery']['skip_rows'] = 1
put_rows(project.task['auth'], project.task['read']['out'], rows)
# if dv360 report
elif project.task['read']['from'] == 'noreply-dv360@google.com':
rows = dv360_report_to_rows(data.getvalue().decode())
rows = dv360_report_clean(rows)
put_rows(project.task['auth'], project.task['read']['out'], rows)
# if csv
elif filename.endswith('.csv'):
rows = csv_to_rows(data.read().decode())
rows = rows_header_sanitize(rows)
put_rows(project.task['auth'], project.task['read']['out'], rows)
else:
if project.verbose: print('UNSUPPORTED FILE:', filename)
put_json('CM_Profile_Roles', PROFILE_ROLES_SCHEMA),
PROFILE_ROLES
)
# Profiles Advertisers
if project.verbose: print('DCM Profile Advertisers')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Advertisers', PROFILE_ADVERTISERS_SCHEMA),
PROFILE_ADVERTISERS
)
if 'subaccounts' in project.task['endpoints']:
# Subaccounts
rows = get_subaccounts(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_SubAccounts', SUBACCOUNTS_SCHEMA),
rows
)
if 'advertisers' in project.task['endpoints']:
# Advertisers
rows = get_advertisers(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Advertisers', ADVERTISERS_SCHEMA),
rows
)
#if 'changelogs' in project.task['endpoints']:
# # Changelogs
put_json('CM_Accounts', ACCOUNTS_SCHEMA),
rows
)
if 'profiles' in project.task['endpoints']:
# Profiles
rows = get_profiles(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Profiles', PROFILES_SCHEMA),
rows
)
# Profiles Campaigns
if project.verbose: print('DCM Profile Campaigns')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Campaigns', PROFILE_CAMPAIGNS_SCHEMA),
PROFILE_CAMPAIGNS
)
# Profiles Sites
if project.verbose: print('DCM Profile Sites')
put_rows(
project.task['out']['auth'],
put_json('CM_Profile_Sites', PROFILE_SITES_SCHEMA),
PROFILE_SITES
)
# Profiles Roles
if project.verbose: print('DCM Profile Roles')
put_rows(
rows = cm_report_clean(rows)
# if bigquery, remove header and determine schema
schema = None
if 'bigquery' in project.task['read']['out']:
schema = cm_report_schema(next(rows))
project.task['read']['out']['bigquery']['schema'] = schema
project.task['read']['out']['bigquery']['skip_rows'] = 1
put_rows(project.task['auth'], project.task['read']['out'], rows)
# if dv360 report
elif project.task['read']['from'] == 'noreply-dv360@google.com':
rows = dv360_report_to_rows(data.getvalue().decode())
rows = dv360_report_clean(rows)
put_rows(project.task['auth'], project.task['read']['out'], rows)
# if csv
elif filename.endswith('.csv'):
rows = csv_to_rows(data.read().decode())
rows = rows_header_sanitize(rows)
put_rows(project.task['auth'], project.task['read']['out'], rows)
else:
if project.verbose: print('UNSUPPORTED FILE:', filename)
def write_to_bigquery(table_id, schema, data, data_format):
out = {
'bigquery':{
'format': data_format,
'dataset': project.task['dataset'],
'table': table_id,
'schema': schema,
'skip_rows': 0,
'disposition': 'WRITE_TRUNCATE'
}
}
put_rows(project.task['auth'], out, data)
# rows
# )
if 'campaigns' in project.task['endpoints']:
# Campaigns
rows = get_campaigns(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Campaigns', CAMPAIGNS_SCHEMA),
rows
)
if 'sites' in project.task['endpoints']:
# Sites
rows = get_sites(accounts)
put_rows(
project.task['out']['auth'],
put_json('CM_Sites', SITES_SCHEMA),
rows
)
# Sites Contacts
if project.verbose: print('DCM Site Contacts')
put_rows(
project.task['out']['auth'],
put_json('CM_Site_Contacts', SITE_CONTACTS_SCHEMA),
SITE_CONTACTS
)
if 'roles' in project.task['endpoints']:
# Roles
rows = get_roles(accounts)