How to use the starthinker.util.project.project.verbose function in starthinker

To help you get started, we’ve selected a few starthinker examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github google / starthinker / starthinker / util / dcm / __init__.py View on Github external
raise Exception('Report does not exist:', name)
    else:
      report_id = report['id']

  running = False

  # zero means run once
  while timeout >= 0:

    # loop all files recent to oldest looking for valid one
    for file_json in report_files(auth, account, report_id):
      #pprint.PrettyPrinter().pprint(file)

      # still running ( wait for timeout )
      if file_json['status'] == 'PROCESSING':
        if project.verbose: print('REPORT PROCESSING WILL WAIT')
        running = True
        if timeout > 0: break # go to outer loop wait

      # ready for download ( return file )
      elif file_json['status'] == 'REPORT_AVAILABLE':
        if project.verbose: print('REPORT DONE')
        return file_json

      # cancelled or failed ( go to next file in loop )

    # if no report running ( skip wait )
    if not running: break

    # sleep a minute
    if timeout > 0:
      if project.verbose: print('WAITING MINUTES', timeout)
github google / starthinker / starthinker / task / dbm / run.py View on Github external
def dbm():
  if project.verbose: print('DBM')

  # name is redundant if title is given, allow skipping use of name for creating reports
  if 'body' in project.task['report'] and 'name' not in project.task['report']:
    project.task['report']['name'] = project.task['report']['body']['metadata']['title']

  # check if report is to be deleted
  if project.task.get('delete', False):
    if project.verbose: print('DBM DELETE', end='')
    report_delete(
      project.task['auth'],
      project.task['report'].get('report_id', None),
      project.task['report'].get('name', None)
    )

  # check if report is to be created
  if 'body' in project.task['report']:
    if project.verbose: print('DBM BUILD', project.task['report']['body']['metadata']['title'])

    # ceck if filters given ( returns new body )
    if 'filters' in project.task['report']:
      project.task['report']['body'] = report_filter(
        project.task['auth'],
        project.task['report']['body'],
        project.task['report']['filters']
github google / starthinker / starthinker / task / mapping / run.py View on Github external
@project.from_parameters
def mapping():
  if project.verbose: print 'MAPPING'

  # create the sheet from template if it does not exist
  sheets_tab_copy(project.task['auth'], TEMPLATE_SHEET, TEMPLATE_TAB, project.task['sheet'], project.task['tab'])

  # move if specified
  dimensions = {}
  defaults = {}
  rows = sheets_read(project.task['auth'], project.task['sheet'], project.task['tab'], 'A1:D')

  # if rows don't exist, query is still created without mapping ( allows blank maps )
  if rows:
    # sanitize mapping
    # 0 = Dimension, 1 = Tag, 2 = Column, 3 = Keyword
    for row in rows[1:]:
      if project.verbose: print 'ROW: ', row
      # sanitize row
github google / starthinker / starthinker / task / sov_dbm / run.py View on Github external
def sov_process_client(report_name):
  sov_rows = {}

  # Download DBM report: ['Advertiser', 'Advertiser ID', 'Advertiser Status', 'Advertiser Integration Code', 'Date', 'Month', 'Device Type', 'Creative Type', 'Region', 'Region ID', 'City', 'City ID', 'Impressions']
  #                        0             1                2                    3                              4       5        6              7                8         9            10      11         12
  filename, report = report_file(
    project.task['auth'],
    None,
    report_name,
    60,
    DBM_CHUNKSIZE
  )

  # if a report exists
  if report:
    if project.verbose: print('CLIENT FILE', filename)

    # convert report to array
    rows = report_to_rows(report)
    rows = report_clean(rows, datastudio=True, nulls=True)
    rows = rows_header_trim(rows)
 
    # pull only needed fields ( see: SCHEMA )
    for row in rows:
      key = ''.join(row[4:-1]) # Everything except impressions

      # if peer is in sov, then just add the impressions
      if key in sov_rows:
        sov_rows[key][7] += int(row[12])

      # otherwise, create a new anonymous peer row
      else:
github google / starthinker / starthinker / task / smartsheet / run.py View on Github external
def smartsheet():
  if project.verbose: print('Smartsheet')

  link = project.task.get('link', True)

  smart = Smartsheet(access_token=project.task['token'])
  smart.errors_as_exceptions(True)

  sheet = smart.Sheets.get_sheet(project.task['sheet'], include=("rowPermalink" if link else ''))
  rows = get_rows(sheet, False, link) 

  project.task['out'].setdefault('bigquery', {}).setdefault('schema', get_schema(sheet))
  print('SCHEMA = %s' % json.dumps(project.task['out']['bigquery']['schema'], indent=2))

  if link and 'schema' in project.task['out']['bigquery']: 
    project.task['out']['bigquery']['schema'].insert(0, { "name":"rowPermalink", "type":"STRING", "mode":"NULLABLE" })

  if rows: put_rows(project.task['auth'], project.task['out'], rows)
github google / starthinker / starthinker / task / barnacle / run.py View on Github external
def get_accounts(accounts):
  if project.verbose: print('DCM Accounts')

  for account_id in accounts:
    is_superuser, profile_id = get_profile_for_api(project.task['auth'], account_id)
    kwargs = { 'profileId':profile_id, 'id':account_id }
    account = API_DCM("user").accounts().get(**kwargs).execute()
    yield [
      account['id'],
      account['name'],
      account['active'],
      account.get('description', ''),
      id_to_timezone(account['reportsConfiguration']['reportGenerationTimeZoneId']),
      account.get('currencyId'),
      account.get('countryId'),
      account['locale'],
      account['nielsenOcrEnabled'],
      account['shareReportsWithTwitter'],
github google / starthinker / starthinker / task / entity / run.py View on Github external
@project.from_parameters
def entity():
  if project.verbose: print 'ENTITY'

  # legacy translations ( changed partners, advertisers to accounts with "partner_id:advertiser_id" )
  if 'partner_id' in project.task:
    project.task['accounts'] = [project.task['partner_id']]

  # create entities
  for entity in project.task['entities']:
    if project.verbose: print 'ENTITY:', entity

    # write public files only once
    if entity in PUBLIC_FILES:
      path = 'gdbm-public:entity/%s.0.%s.json' % (project.date.strftime('%Y%m%d'), entity)
      schema = Entity_Schema_Lookup[entity]
      move_entity(project, path, entity, schema, 'WRITE_TRUNCATE')

    # supports multiple partners, first one resets table, others append
github google / starthinker / starthinker / util / drive / __init__.py View on Github external
elif url_or_name.startswith('https://datastudio.google.com/'):
    m = re.search('^(?:https:\/\/datastudio.google.com\/c\/\w+\/)([a-zA-Z0-9-_]+)(?:\/.*)?$', url_or_name)
    if m: return m.group(1)

  # check if name given convert to ID "Some Document"
  else:
    document = file_find(auth, url_or_name)
    if document: return document['id']

    # check if just ID given, "1uN9tnb-DZ9zZflZsoW4_34sf34tw3ff"
    else:
      m = re.search('^([a-zA-Z0-9-_]+)$', url_or_name)
      if m: return m.group(1)

  # probably a mangled id or name does not exist
  if project.verbose: print('DOCUMENT DOES NOT EXIST', url_or_name)
  return None
github google / starthinker / starthinker / task / dt / run.py View on Github external
def dt_move_small(dt_file, dt_partition, jobs):
  if project.verbose: print("DT TO TABLE SMALL", dt_partition)

  jobs.append(storage_to_table(
    project.task['auth'],
    project.id,
    project.task['to']['dataset'],
    dt_partition,
    '%s:%s*' % (project.task['bucket'], dt_file),
    dt_schema(dt_header(dt_file)),
    1,
    'CSV',
    'WRITE_TRUNCATE',
    False
  ))
github google / starthinker / util / csv / __init__.py View on Github external
def rows_to_csv(rows):
  csv_string = StringIO()
  writer = csv.writer(csv_string, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
  count = 0
  for row_number, row in enumerate(rows):
    try:
      writer.writerow(row)
      count += 1
    except Exception, e:
      print 'Error:', row_number, str(e), row
  csv_string.seek(0)  # important otherwise contents is zero
  if project.verbose: print 'CSV Rows Written:', count
  return csv_string