How to use the starthinker.util.bigquery.query_to_rows function in starthinker

To help you get started, we’ve selected a few starthinker examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github google / starthinker / starthinker / task / video / run.py View on Github external
def videos_from_bigquery(bigquery):
  rows = query_to_rows(project.task['auth'], bigquery.get('project', project.id), bigquery['dataset'], 'SELECT * FROM %s;' % bigquery['table'])
  return rows_to_videos(rows)
github google / starthinker / starthinker / task / marketing / run.py View on Github external
def get_solutions():
  if project.verbose: print 'GETTING SCORES'

  for solution in project.task['solutions']:
    scores = []

    if 'sheet' in solution:
      scores = sheets_read(
        project.task['auth'],
        solution['sheet']['url'],
        solution['sheet']['tab'],
        solution['sheet']['range']
      )
    elif 'bigquery' in solution:
      scores = query_to_rows(
        project.task['auth'],
        project.id,
        solution['bigquery']['dataset'],
        solution['bigquery']['query']
      )

    # for easy lookup use dictionary
    solution['scores'] = {}
    for score in scores:
      solution['scores'].setdefault(str(score[0]), [])
      solution['scores'][str(score[0])].append({ 'variant_id':str(score[1]), 'variant':score[2], 'score':float(score[3]) })

    if project.verbose: print 'GOT SCORES:', len(solution['scores'])

  return project.task['solutions']
github google / starthinker / starthinker / task / marketing / run.py View on Github external
def get_impacts():
  if project.verbose: print 'GETTING IMPACTS'

  impacts = []

  if 'sheet' in project.task['impacts']:
    impacts = sheets_read(
      project.task['auth'],
      project.task['impacts']['sheet']['url'],
      project.task['impacts']['sheet']['tab'],
      project.task['impacts']['sheet']['range']
    )
  elif 'bigquery' in project.task['impacts']:
    impacts = query_to_rows(
      project.task['auth'],
      project.id,
      project.task['impacts']['bigquery']['dataset'],
      project.task['impacts']['bigquery']['query']
    )

  # for easy lookup use dictionary
  impacts = dict([(str(i[0]), float(i[1])) for i in impacts])

  if project.verbose: print 'GOT IMPACTS:', len(impacts)

  return impacts
github google / starthinker / starthinker / task / itp_audit / run.py View on Github external
dataset_id=project.task['dataset'],
    table_id=DV360_CUSTOM_SEGMENTS_SHEET_TABLE,
    rows=sheet_rows,
    schema=schema,
    skip_rows=1,
    disposition='WRITE_TRUNCATE'
    )

  # Run Query
  if project.verbose: print('RUN DV360 Custom Segments Query')
  run_query_from_file(Queries.dv360_custom_segments, 
    project.id, project.task['dataset'], DV360_CUSTOM_SEGMENTS_TABLE)

  # Move Table back to sheets
  query = 'SELECT * from `' + project.id + '.' + project.task['dataset'] + '.' + DV360_CUSTOM_SEGMENTS_TABLE + '`'
  rows = query_to_rows('service', project.id, project.task['dataset'], query, legacy=False)

  # makes sure types are correct in sheet
  a1_notation = a1_notation[:1] + '2' +a1_notation[1:]
  rows = rows_to_type(rows)
  sheets_clear('user', project.task['sheet'], 'DV3 Segments', a1_notation)
  sheets_write('user', project.task['sheet'], 'DV3 Segments', a1_notation, rows)
github google / starthinker / starthinker / task / bigquery / run.py View on Github external
query_to_table(
        project.task['auth'],
        project.id,
        project.task['to']['dataset'],
        project.task['to']['table'],
        query_parameters(project.task['from']['query'], project.task['from'].get('parameters')),
        disposition = project.task['write_disposition'] if 'write_disposition' in project.task else 'WRITE_TRUNCATE',
        legacy=project.task['from'].get('legacy', project.task['from'].get('useLegacySql', True)), # DEPRECATED: useLegacySql,
        target_project_id=project.task['to'].get('project_id', project.id)
      )
    # NOT USED SO RIPPING IT OUT
    # Mauriciod: Yes, it is used, look at project/mauriciod/target_winrate.json
    elif 'storage' in project.task['to']:
      if project.verbose: print("QUERY TO STORAGE", project.task['to']['storage'])
      local_file_name = '/tmp/%s' % str(uuid.uuid1())
      rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'])

      f = open(local_file_name, 'wb')
      writer = csv.writer(f)
      writer.writerows(rows)
      f.close()

      f = open(local_file_name, 'rb')
      object_put(project.task['auth'], project.task['to']['storage'], f)
      f.close()

      os.remove(local_file_name)
    elif 'sheet' in project.task['to']:
      if project.verbose: print("QUERY TO SHEET", project.task['to']['sheet'])
      rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'], legacy=project.task['from'].get('legacy', True))

      # makes sure types are correct in sheet
github google / starthinker / starthinker / task / itp_audit / run.py View on Github external
auth='service',
    project_id=project.id,
    dataset_id=project.task['dataset'],
    table_id=CM_SITE_SEGMENTATION_SHEET_TABLE,
    rows=sheet_rows,
    schema=schema,
    skip_rows=1,
    disposition='WRITE_TRUNCATE'
  )

  # Get Site_Type from the sheet
  run_query_from_file(Queries.cm_site_segmentation, project.id, project.task['dataset'], CM_SITE_SEGMENTATION_TABLE)  

  # Move Table back to sheets
  query = 'SELECT * from `' + project.id + '.' + project.task['dataset'] + '.' + CM_SITE_SEGMENTATION_TABLE + '`'
  rows = query_to_rows('service', project.id, project.task['dataset'], query, legacy=False)

  # makes sure types are correct in sheet
  rows = rows_to_type(rows)
  sheets_clear('user', project.task['sheet'], 'CM_Site_Segments', 'A2:C')
  sheets_write('user', project.task['sheet'], 'CM_Site_Segments', 'A2:C', rows)
github google / starthinker / starthinker / task / bigquery / run.py View on Github external
f = open(local_file_name, 'rb')
      object_put(project.task['auth'], project.task['to']['storage'], f)
      f.close()

      os.remove(local_file_name)
    elif 'sheet' in project.task['to']:
      if project.verbose: print("QUERY TO SHEET", project.task['to']['sheet'])
      rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'], legacy=project.task['from'].get('legacy', True))

      # makes sure types are correct in sheet
      rows = rows_to_type(rows)

      sheets_clear(project.task['auth'], project.task['to']['sheet'], project.task['to']['tab'], project.task['to'].get('range', 'A2'))
      sheets_write(project.task['auth'], project.task['to']['sheet'], project.task['to']['tab'], project.task['to'].get('range', 'A2'), rows)
    elif 'sftp' in project.task['to']:
      rows = query_to_rows(project.task['auth'], project.id, project.task['from']['dataset'], project.task['from']['query'], legacy=project.task['from'].get('use_legacy_sql', True))

      if rows:
        if project.verbose: print("QUERY TO SFTP")
        put_rows(project.task['auth'], project.task['to'], rows)
    else:
      if project.verbose: print("QUERY TO VIEW", project.task['to']['view'])
      query_to_view(
        project.task['auth'],
        project.id,
        project.task['to']['dataset'],
        project.task['to']['view'],
        query_parameters(project.task['from']['query'], project.task['from'].get('parameters')),
        project.task['from'].get('legacy', project.task['from'].get('useLegacySql', True)), # DEPRECATED: useLegacySql
        project.task['to'].get('replace', False)
      )
  else: