Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def videos_from_bigquery(bigquery):
rows = query_to_rows(project.task['auth'], bigquery.get('project', project.id), bigquery['dataset'], 'SELECT * FROM %s;' % bigquery['table'])
return rows_to_videos(rows)
def get_solutions():
if project.verbose: print 'GETTING SCORES'
for solution in project.task['solutions']:
scores = []
if 'sheet' in solution:
scores = sheets_read(
project.task['auth'],
solution['sheet']['url'],
solution['sheet']['tab'],
solution['sheet']['range']
)
elif 'bigquery' in solution:
scores = query_to_rows(
project.task['auth'],
project.id,
solution['bigquery']['dataset'],
solution['bigquery']['query']
)
# for easy lookup use dictionary
solution['scores'] = {}
for score in scores:
solution['scores'].setdefault(str(score[0]), [])
solution['scores'][str(score[0])].append({ 'variant_id':str(score[1]), 'variant':score[2], 'score':float(score[3]) })
if project.verbose: print 'GOT SCORES:', len(solution['scores'])
return project.task['solutions']
def get_impacts():
if project.verbose: print 'GETTING IMPACTS'
impacts = []
if 'sheet' in project.task['impacts']:
impacts = sheets_read(
project.task['auth'],
project.task['impacts']['sheet']['url'],
project.task['impacts']['sheet']['tab'],
project.task['impacts']['sheet']['range']
)
elif 'bigquery' in project.task['impacts']:
impacts = query_to_rows(
project.task['auth'],
project.id,
project.task['impacts']['bigquery']['dataset'],
project.task['impacts']['bigquery']['query']
)
# for easy lookup use dictionary
impacts = dict([(str(i[0]), float(i[1])) for i in impacts])
if project.verbose: print 'GOT IMPACTS:', len(impacts)
return impacts
dataset_id=project.task['dataset'],
table_id=DV360_CUSTOM_SEGMENTS_SHEET_TABLE,
rows=sheet_rows,
schema=schema,
skip_rows=1,
disposition='WRITE_TRUNCATE'
)
# Run Query
if project.verbose: print('RUN DV360 Custom Segments Query')
run_query_from_file(Queries.dv360_custom_segments,
project.id, project.task['dataset'], DV360_CUSTOM_SEGMENTS_TABLE)
# Move Table back to sheets
query = 'SELECT * from `' + project.id + '.' + project.task['dataset'] + '.' + DV360_CUSTOM_SEGMENTS_TABLE + '`'
rows = query_to_rows('service', project.id, project.task['dataset'], query, legacy=False)
# makes sure types are correct in sheet
a1_notation = a1_notation[:1] + '2' +a1_notation[1:]
rows = rows_to_type(rows)
sheets_clear('user', project.task['sheet'], 'DV3 Segments', a1_notation)
sheets_write('user', project.task['sheet'], 'DV3 Segments', a1_notation, rows)
query_to_table(
project.task['auth'],
project.id,
project.task['to']['dataset'],
project.task['to']['table'],
query_parameters(project.task['from']['query'], project.task['from'].get('parameters')),
disposition = project.task['write_disposition'] if 'write_disposition' in project.task else 'WRITE_TRUNCATE',
legacy=project.task['from'].get('legacy', project.task['from'].get('useLegacySql', True)), # DEPRECATED: useLegacySql,
target_project_id=project.task['to'].get('project_id', project.id)
)
# NOT USED SO RIPPING IT OUT
# Mauriciod: Yes, it is used, look at project/mauriciod/target_winrate.json
elif 'storage' in project.task['to']:
if project.verbose: print("QUERY TO STORAGE", project.task['to']['storage'])
local_file_name = '/tmp/%s' % str(uuid.uuid1())
rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'])
f = open(local_file_name, 'wb')
writer = csv.writer(f)
writer.writerows(rows)
f.close()
f = open(local_file_name, 'rb')
object_put(project.task['auth'], project.task['to']['storage'], f)
f.close()
os.remove(local_file_name)
elif 'sheet' in project.task['to']:
if project.verbose: print("QUERY TO SHEET", project.task['to']['sheet'])
rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'], legacy=project.task['from'].get('legacy', True))
# makes sure types are correct in sheet
auth='service',
project_id=project.id,
dataset_id=project.task['dataset'],
table_id=CM_SITE_SEGMENTATION_SHEET_TABLE,
rows=sheet_rows,
schema=schema,
skip_rows=1,
disposition='WRITE_TRUNCATE'
)
# Get Site_Type from the sheet
run_query_from_file(Queries.cm_site_segmentation, project.id, project.task['dataset'], CM_SITE_SEGMENTATION_TABLE)
# Move Table back to sheets
query = 'SELECT * from `' + project.id + '.' + project.task['dataset'] + '.' + CM_SITE_SEGMENTATION_TABLE + '`'
rows = query_to_rows('service', project.id, project.task['dataset'], query, legacy=False)
# makes sure types are correct in sheet
rows = rows_to_type(rows)
sheets_clear('user', project.task['sheet'], 'CM_Site_Segments', 'A2:C')
sheets_write('user', project.task['sheet'], 'CM_Site_Segments', 'A2:C', rows)
f = open(local_file_name, 'rb')
object_put(project.task['auth'], project.task['to']['storage'], f)
f.close()
os.remove(local_file_name)
elif 'sheet' in project.task['to']:
if project.verbose: print("QUERY TO SHEET", project.task['to']['sheet'])
rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'], legacy=project.task['from'].get('legacy', True))
# makes sure types are correct in sheet
rows = rows_to_type(rows)
sheets_clear(project.task['auth'], project.task['to']['sheet'], project.task['to']['tab'], project.task['to'].get('range', 'A2'))
sheets_write(project.task['auth'], project.task['to']['sheet'], project.task['to']['tab'], project.task['to'].get('range', 'A2'), rows)
elif 'sftp' in project.task['to']:
rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'], legacy=project.task['from'].get('use_legacy_sql', True))
if rows:
if project.verbose: print("QUERY TO SFTP")
put_rows(project.task['auth'], project.task['to'], rows)
else:
if project.verbose: print("QUERY TO VIEW", project.task['to']['view'])
query_to_view(
project.task['auth'],
project.id,
project.task['to']['dataset'],
project.task['to']['view'],
query_parameters(project.task['from']['query'], project.task['from'].get('parameters')),
project.task['from'].get('legacy', project.task['from'].get('useLegacySql', True)), # DEPRECATED: useLegacySql
project.task['to'].get('replace', False)
)
else: