Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
raise Exception('Report does not exist:', name)
else:
report_id = report['id']
running = False
# zero means run once
while timeout >= 0:
# loop all files recent to oldest looking for valid one
for file_json in report_files(auth, account, report_id):
#pprint.PrettyPrinter().pprint(file)
# still running ( wait for timeout )
if file_json['status'] == 'PROCESSING':
if project.verbose: print('REPORT PROCESSING WILL WAIT')
running = True
if timeout > 0: break # go to outer loop wait
# ready for download ( return file )
elif file_json['status'] == 'REPORT_AVAILABLE':
if project.verbose: print('REPORT DONE')
return file_json
# cancelled or failed ( go to next file in loop )
# if no report running ( skip wait )
if not running: break
# sleep a minute
if timeout > 0:
if project.verbose: print('WAITING MINUTES', timeout)
def dbm():
if project.verbose: print('DBM')
# name is redundant if title is given, allow skipping use of name for creating reports
if 'body' in project.task['report'] and 'name' not in project.task['report']:
project.task['report']['name'] = project.task['report']['body']['metadata']['title']
# check if report is to be deleted
if project.task.get('delete', False):
if project.verbose: print('DBM DELETE', end='')
report_delete(
project.task['auth'],
project.task['report'].get('report_id', None),
project.task['report'].get('name', None)
)
# check if report is to be created
if 'body' in project.task['report']:
if project.verbose: print('DBM BUILD', project.task['report']['body']['metadata']['title'])
# ceck if filters given ( returns new body )
if 'filters' in project.task['report']:
project.task['report']['body'] = report_filter(
project.task['auth'],
project.task['report']['body'],
project.task['report']['filters']
@project.from_parameters
def mapping():
if project.verbose: print 'MAPPING'
# create the sheet from template if it does not exist
sheets_tab_copy(project.task['auth'], TEMPLATE_SHEET, TEMPLATE_TAB, project.task['sheet'], project.task['tab'])
# move if specified
dimensions = {}
defaults = {}
rows = sheets_read(project.task['auth'], project.task['sheet'], project.task['tab'], 'A1:D')
# if rows don't exist, query is still created without mapping ( allows blank maps )
if rows:
# sanitize mapping
# 0 = Dimension, 1 = Tag, 2 = Column, 3 = Keyword
for row in rows[1:]:
if project.verbose: print 'ROW: ', row
# sanitize row
def sov_process_client(report_name):
sov_rows = {}
# Download DBM report: ['Advertiser', 'Advertiser ID', 'Advertiser Status', 'Advertiser Integration Code', 'Date', 'Month', 'Device Type', 'Creative Type', 'Region', 'Region ID', 'City', 'City ID', 'Impressions']
# 0 1 2 3 4 5 6 7 8 9 10 11 12
filename, report = report_file(
project.task['auth'],
None,
report_name,
60,
DBM_CHUNKSIZE
)
# if a report exists
if report:
if project.verbose: print('CLIENT FILE', filename)
# convert report to array
rows = report_to_rows(report)
rows = report_clean(rows, datastudio=True, nulls=True)
rows = rows_header_trim(rows)
# pull only needed fields ( see: SCHEMA )
for row in rows:
key = ''.join(row[4:-1]) # Everything except impressions
# if peer is in sov, then just add the impressions
if key in sov_rows:
sov_rows[key][7] += int(row[12])
# otherwise, create a new anonymous peer row
else:
def smartsheet():
if project.verbose: print('Smartsheet')
link = project.task.get('link', True)
smart = Smartsheet(access_token=project.task['token'])
smart.errors_as_exceptions(True)
sheet = smart.Sheets.get_sheet(project.task['sheet'], include=("rowPermalink" if link else ''))
rows = get_rows(sheet, False, link)
project.task['out'].setdefault('bigquery', {}).setdefault('schema', get_schema(sheet))
print('SCHEMA = %s' % json.dumps(project.task['out']['bigquery']['schema'], indent=2))
if link and 'schema' in project.task['out']['bigquery']:
project.task['out']['bigquery']['schema'].insert(0, { "name":"rowPermalink", "type":"STRING", "mode":"NULLABLE" })
if rows: put_rows(project.task['auth'], project.task['out'], rows)
def get_accounts(accounts):
if project.verbose: print('DCM Accounts')
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'], account_id)
kwargs = { 'profileId':profile_id, 'id':account_id }
account = API_DCM("user").accounts().get(**kwargs).execute()
yield [
account['id'],
account['name'],
account['active'],
account.get('description', ''),
id_to_timezone(account['reportsConfiguration']['reportGenerationTimeZoneId']),
account.get('currencyId'),
account.get('countryId'),
account['locale'],
account['nielsenOcrEnabled'],
account['shareReportsWithTwitter'],
@project.from_parameters
def entity():
if project.verbose: print 'ENTITY'
# legacy translations ( changed partners, advertisers to accounts with "partner_id:advertiser_id" )
if 'partner_id' in project.task:
project.task['accounts'] = [project.task['partner_id']]
# create entities
for entity in project.task['entities']:
if project.verbose: print 'ENTITY:', entity
# write public files only once
if entity in PUBLIC_FILES:
path = 'gdbm-public:entity/%s.0.%s.json' % (project.date.strftime('%Y%m%d'), entity)
schema = Entity_Schema_Lookup[entity]
move_entity(project, path, entity, schema, 'WRITE_TRUNCATE')
# supports multiple partners, first one resets table, others append
elif url_or_name.startswith('https://datastudio.google.com/'):
m = re.search('^(?:https:\/\/datastudio.google.com\/c\/\w+\/)([a-zA-Z0-9-_]+)(?:\/.*)?$', url_or_name)
if m: return m.group(1)
# check if name given convert to ID "Some Document"
else:
document = file_find(auth, url_or_name)
if document: return document['id']
# check if just ID given, "1uN9tnb-DZ9zZflZsoW4_34sf34tw3ff"
else:
m = re.search('^([a-zA-Z0-9-_]+)$', url_or_name)
if m: return m.group(1)
# probably a mangled id or name does not exist
if project.verbose: print('DOCUMENT DOES NOT EXIST', url_or_name)
return None
def dt_move_small(dt_file, dt_partition, jobs):
if project.verbose: print("DT TO TABLE SMALL", dt_partition)
jobs.append(storage_to_table(
project.task['auth'],
project.id,
project.task['to']['dataset'],
dt_partition,
'%s:%s*' % (project.task['bucket'], dt_file),
dt_schema(dt_header(dt_file)),
1,
'CSV',
'WRITE_TRUNCATE',
False
))
def rows_to_csv(rows):
csv_string = StringIO()
writer = csv.writer(csv_string, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
count = 0
for row_number, row in enumerate(rows):
try:
writer.writerow(row)
count += 1
except Exception, e:
print 'Error:', row_number, str(e), row
csv_string.seek(0) # important otherwise contents is zero
if project.verbose: print 'CSV Rows Written:', count
return csv_string