How to use the starthinker.util.data.put_rows function in starthinker

To help you get started, we’ve selected a few starthinker examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github google / starthinker / starthinker / task / barnacle / run.py View on Github external
SITE_CONTACTS
    )

  if 'roles' in project.task['endpoints']:
    # Roles
    rows = get_roles(accounts)
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Roles', ROLES_SCHEMA),
      rows
    )
  
  if 'reports' in project.task['endpoints'] or project.task.get('reports') == True:
    # Reports
    rows = get_reports(accounts)
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Reports', REPORTS_SCHEMA),
      rows
    )

    # Reports Deliveries
    if project.verbose: print('DCM Deliveries')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Report_Deliveries', REPORT_DELIVERIES_SCHEMA),
      REPORT_DELIVERIES
    )
github google / starthinker / starthinker / task / barnacle / run.py View on Github external
accounts = set(get_rows("user", project.task['accounts']))

  if 'accounts' in project.task['endpoints']:
    # Accounts
    rows = get_accounts(accounts)
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Accounts', ACCOUNTS_SCHEMA),
      rows
    )

  if 'profiles' in project.task['endpoints']:
    # Profiles
    rows = get_profiles(accounts)
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profiles', PROFILES_SCHEMA),
      rows
    )

    # Profiles Campaigns
    if project.verbose: print('DCM Profile Campaigns')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Campaigns', PROFILE_CAMPAIGNS_SCHEMA),
      PROFILE_CAMPAIGNS
    )

    # Profiles Sites
    if project.verbose: print('DCM Profile Sites')
    put_rows(
github google / starthinker / starthinker / task / barnacle / run.py View on Github external
project.task['out']['auth'], 
      put_json('CM_Profiles', PROFILES_SCHEMA),
      rows
    )

    # Profiles Campaigns
    if project.verbose: print('DCM Profile Campaigns')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Campaigns', PROFILE_CAMPAIGNS_SCHEMA),
      PROFILE_CAMPAIGNS
    )

    # Profiles Sites
    if project.verbose: print('DCM Profile Sites')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Sites', PROFILE_SITES_SCHEMA),
      PROFILE_SITES
    )

    # Profiles Roles
    if project.verbose: print('DCM Profile Roles')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Roles', PROFILE_ROLES_SCHEMA),
      PROFILE_ROLES
    )

    # Profiles Advertisers
    if project.verbose: print('DCM Profile Advertisers')
    put_rows(
github google / starthinker / task / twitter / run.py View on Github external
if 'trends' in project.task:
    if 'places' in project.task['trends']: 
      rows = twitter_trends_places()
      project.task['out']['bigquery']['schema'] = TWITTER_TRENDS_PLACE_SCHEMA
      project.task['out']['bigquery']['skip_rows'] = 0
    elif 'closest' in project.task['trends']: 
      rows = twitter_trends_closest()
      project.task['out']['bigquery']['schema'] = TWITTER_TRENDS_CLOSEST_SCHEMA
      project.task['out']['bigquery']['skip_rows'] = 0
    else: 
      rows = twitter_trends_available()
      project.task['out']['bigquery']['schema'] = TWITTER_TRENDS_AVAILABLE_SCHEMA
      project.task['out']['bigquery']['skip_rows'] = 0

  if rows: put_rows(project.task['auth'], project.task['out'], 'twitter_%s.csv' % project.date, rows)
github google / starthinker / starthinker / task / email / run.py View on Github external
project.task['read']['out']['bigquery']['schema'] = schema
        project.task['read']['out']['bigquery']['skip_rows'] = 1

      put_rows(project.task['auth'], project.task['read']['out'], rows)

    # if dv360 report
    elif project.task['read']['from'] == 'noreply-dv360@google.com':
      rows = dv360_report_to_rows(data.getvalue().decode())
      rows = dv360_report_clean(rows)
      put_rows(project.task['auth'], project.task['read']['out'], rows)

    # if csv
    elif filename.endswith('.csv'):
      rows = csv_to_rows(data.read().decode())
      rows = rows_header_sanitize(rows)
      put_rows(project.task['auth'], project.task['read']['out'], rows)

    else:
      if project.verbose: print('UNSUPPORTED FILE:', filename)
github google / starthinker / starthinker / task / barnacle / run.py View on Github external
put_json('CM_Profile_Roles', PROFILE_ROLES_SCHEMA),
      PROFILE_ROLES
    )

    # Profiles Advertisers
    if project.verbose: print('DCM Profile Advertisers')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Advertisers', PROFILE_ADVERTISERS_SCHEMA),
      PROFILE_ADVERTISERS
    )

  if 'subaccounts' in project.task['endpoints']:
    # Subaccounts
    rows = get_subaccounts(accounts)
    put_rows(
      project.task['out']['auth'],
      put_json('CM_SubAccounts', SUBACCOUNTS_SCHEMA),
      rows
    )

  if 'advertisers' in project.task['endpoints']:
    # Advertisers
    rows = get_advertisers(accounts)
    put_rows(
      project.task['out']['auth'],
      put_json('CM_Advertisers', ADVERTISERS_SCHEMA),
      rows
    )

  #if 'changelogs' in project.task['endpoints']:
  #  # Changelogs 
github google / starthinker / starthinker / task / barnacle / run.py View on Github external
put_json('CM_Accounts', ACCOUNTS_SCHEMA),
      rows
    )

  if 'profiles' in project.task['endpoints']:
    # Profiles
    rows = get_profiles(accounts)
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profiles', PROFILES_SCHEMA),
      rows
    )

    # Profiles Campaigns
    if project.verbose: print('DCM Profile Campaigns')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Campaigns', PROFILE_CAMPAIGNS_SCHEMA),
      PROFILE_CAMPAIGNS
    )

    # Profiles Sites
    if project.verbose: print('DCM Profile Sites')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Profile_Sites', PROFILE_SITES_SCHEMA),
      PROFILE_SITES
    )

    # Profiles Roles
    if project.verbose: print('DCM Profile Roles')
    put_rows(
github google / starthinker / starthinker / task / email / run.py View on Github external
rows = cm_report_clean(rows)

      # if bigquery, remove header and determine schema
      schema = None
      if 'bigquery' in project.task['read']['out']:
        schema = cm_report_schema(next(rows))
        project.task['read']['out']['bigquery']['schema'] = schema
        project.task['read']['out']['bigquery']['skip_rows'] = 1

      put_rows(project.task['auth'], project.task['read']['out'], rows)

    # if dv360 report
    elif project.task['read']['from'] == 'noreply-dv360@google.com':
      rows = dv360_report_to_rows(data.getvalue().decode())
      rows = dv360_report_clean(rows)
      put_rows(project.task['auth'], project.task['read']['out'], rows)

    # if csv
    elif filename.endswith('.csv'):
      rows = csv_to_rows(data.read().decode())
      rows = rows_header_sanitize(rows)
      put_rows(project.task['auth'], project.task['read']['out'], rows)

    else:
      if project.verbose: print('UNSUPPORTED FILE:', filename)
github google / starthinker / starthinker / task / ga_settings_download / run.py View on Github external
def write_to_bigquery(table_id, schema, data, data_format):
  out = {
   'bigquery':{
      'format': data_format,
      'dataset': project.task['dataset'],
      'table': table_id,
      'schema': schema,
      'skip_rows': 0,
      'disposition': 'WRITE_TRUNCATE'
    }
  }
  put_rows(project.task['auth'], out, data)
github google / starthinker / starthinker / task / barnacle / run.py View on Github external
#    rows
  #  )

  if 'campaigns' in project.task['endpoints']:
    # Campaigns 
    rows = get_campaigns(accounts)
    put_rows(
      project.task['out']['auth'],
      put_json('CM_Campaigns', CAMPAIGNS_SCHEMA),
      rows
    )

  if 'sites' in project.task['endpoints']:
    # Sites 
    rows = get_sites(accounts)
    put_rows(
      project.task['out']['auth'],
      put_json('CM_Sites', SITES_SCHEMA),
      rows
    )
  
    # Sites Contacts
    if project.verbose: print('DCM Site Contacts')
    put_rows(
      project.task['out']['auth'], 
      put_json('CM_Site_Contacts', SITE_CONTACTS_SCHEMA),
      SITE_CONTACTS
    )

  if 'roles' in project.task['endpoints']:
    # Roles
    rows = get_roles(accounts)