Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def report_fetch(auth, report_id):
service = get_service('doubleclicksearch', 'v2', auth)
if project.verbose: print 'Fetching Report', report_id
files = report_ready(service, report_id)
reports = []
i = 0
for file in files:
reports.append({'name': '%s_%d_%s.csv' % (report_id, i, str(date.today())), 'report_id': report_id, 'report_fragment': i})
return reports
input_file_name = uncompressed_file
input_file = open(input_file_name, 'rb')
reader = csv.reader(input_file)
header = next(reader)
input_file.seek(0)
schema = field_list_to_schema(header)
output_file_name = '/tmp/%s.csv' % str(uuid.uuid1())
processor.clean_csv(input_file, output_file_name, len(header), header=True)
input_file.close()
output_file = open(output_file_name, 'rb')
io_to_table(
project.task['auth'],
project.id,
project.task['to'].get('dataset'),
project.task['to'].get('table'),
output_file,
'CSV',
schema,
skip_rows=0,
disposition=project.task['to'].get('write_disposition',
'WRITE_TRUNCATE')
)
output_file.close()
os.remove(input_file_name)
os.remove(output_file_name)
def sheets():
if project.verbose: print('SHEETS')
# if sheet or tab is missing, don't do anything
if not project.task.get('sheet') or not project.task.get('tab'):
if project.verbose: print('Missing Sheet and/or Tab, skipping task.')
return
# delete if specified, will delete sheet if no more tabs remain
if project.task.get('delete', False):
sheets_tab_delete(
project.task['auth'],
project.task['sheet'],
project.task['tab']
)
# create a sheet and tab if specified, if template
if 'template' in project.task:
def floodlight_report(floodlight_id):
account_id, subaccount_id = parse_account(project.task['auth'], project.task['account'])
name = 'Floodlight Monitor %s %s ( StarThinker )' % ( account_id, floodlight_id )
if project.verbose: print("FLOODLIGHT MONITOR REPORT: ", name)
# create report if it does not exists
report = report_build(
project.task['auth'],
project.task['account'],
{
'kind': 'dfareporting#report',
'type': 'FLOODLIGHT',
'accountId': account_id,
'name': name,
'fileName': name.replace('( ', '').replace(' )', '').replace(' ', '_'),
'format': 'CSV',
def custom_metrics_download(accounts, current_date):
if project.verbose: print('Downloading GA Custom Metrics')
for account in accounts:
for web_property in account.get('webProperties', []):
for custom_metric in API_Analytics(project.task['auth'], iterate=True).management().customMetrics().list(accountId=account.get('id'), webPropertyId=web_property.get('id')).execute():
yield (
account.get('name'),
account.get('id'),
web_property.get('name'),
web_property.get('id'),
custom_metric.get('id'),
custom_metric.get('name'),
custom_metric.get('index'),
custom_metric.get('scope'),
custom_metric.get('active'),
custom_metric.get('created'),
custom_metric.get('updated'),
def sdf():
if project.verbose: print('SDF')
# Download sdf files
sdf_zip_file = sdf_download(
project.task['auth'],
project.task['version'],
project.task['partner_id'],
project.task['file_types'],
project.task['filter_type'],
project.task['read']['filter_ids'])
# Load data into BigQuery
sdf_to_bigquery(sdf_zip_file, project.id, project.task['dataset'], project.task['time_partitioned_table'], project.task['create_single_day_table'], project.task.get('table_suffix', ''))
def dt_move_large(dt_file, dt_partition, jobs):
if project.verbose: print("DT TO TABLE LARGE", dt_partition)
delimiter = '\n'
disposition = 'WRITE_TRUNCATE'
# decompression handler for gzip ( must be outside of chunks as it keeps track of stream across multiple calls )
gz_handler = zlib.decompressobj(32 + zlib.MAX_WBITS)
# sliding view of data flowing out of decompression, used to buffer and delimit rows
first_row = True
view = ''
# loop all chunks of file, decompress, and find row delimiter
for data_gz in object_get_chunks(project.task['auth'], '%s:%s' % (project.task['bucket'], dt_file)):
view += gz_handler.decompress(data_gz.read()).decode()
@project.from_parameters
def ga_settings_download():
if project.verbose: print('Initiating GA Settings Download')
filters = [str(f) for f in project.task.get('accounts', [])] # in case integer list is provided
accounts = [a for a in API_Analytics(project.task['auth'], iterate=True).management().accountSummaries().list().execute() if not filters or a['id'] in filters or a['name'] in filters]
if accounts:
current_date = project.date
write_to_bigquery(
'ga_custom_dimension_settings',
CUSTOM_DIMENSION_SCHEMA,
custom_dimensions_download(accounts, current_date),
'CSV'
)
def sheets_read(auth, sheet_url, sheet_tab, sheet_range, retries=10):
if project.verbose: print 'SHEETS READ', sheet_url, sheet_tab, sheet_range
service = get_service('sheets', 'v4', auth)
sheet_id = sheets_id(sheet_url)
return API_Retry(service.spreadsheets().values().get(spreadsheetId=sheet_id, range=sheets_tab_range(sheet_tab, sheet_range)), 'values', retries=retries)