How to use the starthinker.util.project.project function in starthinker

To help you get started, we’ve selected a few starthinker examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github google / starthinker / util / ds / __init__.py View on Github external
def report_fetch(auth, report_id):
  service = get_service('doubleclicksearch', 'v2', auth)
  
  if project.verbose: print 'Fetching Report', report_id
  files = report_ready(service, report_id)
  
  reports = []
  i = 0
  for file in files:
    reports.append({'name': '%s_%d_%s.csv' % (report_id, i, str(date.today())), 'report_id': report_id, 'report_fragment': i})
   
  return reports
github google / starthinker / starthinker / task / sftp / run.py View on Github external
input_file_name = uncompressed_file

    input_file = open(input_file_name, 'rb')

    reader = csv.reader(input_file)
    header = next(reader)
    input_file.seek(0)
    schema = field_list_to_schema(header)
    output_file_name = '/tmp/%s.csv' % str(uuid.uuid1())
    processor.clean_csv(input_file, output_file_name, len(header), header=True)
    input_file.close()

    output_file = open(output_file_name, 'rb')
    io_to_table(
      project.task['auth'],
      project.id,
      project.task['to'].get('dataset'),
      project.task['to'].get('table'),
      output_file,
      'CSV',
      schema,
      skip_rows=0,
      disposition=project.task['to'].get('write_disposition',
      'WRITE_TRUNCATE')
    )
    output_file.close()

    os.remove(input_file_name)

  os.remove(output_file_name)
github google / starthinker / starthinker / task / sheets / run.py View on Github external
def sheets():
  if project.verbose: print('SHEETS')

  # if sheet or tab is missing, don't do anything
  if not project.task.get('sheet') or not project.task.get('tab'):
    if project.verbose: print('Missing Sheet and/or Tab, skipping task.')
    return

  # delete if specified, will delete sheet if no more tabs remain 
  if project.task.get('delete', False):
    sheets_tab_delete(
      project.task['auth'], 
      project.task['sheet'], 
      project.task['tab']
    )

  # create a sheet and tab if specified, if template
  if 'template' in project.task:
github google / starthinker / starthinker / task / floodlight_monitor / run.py View on Github external
def floodlight_report(floodlight_id):

  account_id, subaccount_id = parse_account(project.task['auth'], project.task['account'])

  name = 'Floodlight Monitor %s %s ( StarThinker )' % ( account_id, floodlight_id )

  if project.verbose: print("FLOODLIGHT MONITOR REPORT: ", name)

  # create report if it does not exists
  report = report_build(
    project.task['auth'], 
    project.task['account'], 
    { 
      'kind': 'dfareporting#report',
      'type': 'FLOODLIGHT',
      'accountId': account_id,
      'name': name,
      'fileName': name.replace('( ', '').replace(' )', '').replace(' ', '_'),
      'format': 'CSV',
github google / starthinker / starthinker / task / ga_settings_download / run.py View on Github external
def custom_metrics_download(accounts, current_date):
  if project.verbose: print('Downloading GA Custom Metrics')

  for account in accounts:
    for web_property in account.get('webProperties', []):
      for custom_metric in API_Analytics(project.task['auth'], iterate=True).management().customMetrics().list(accountId=account.get('id'), webPropertyId=web_property.get('id')).execute():
        yield (
          account.get('name'),
          account.get('id'),
          web_property.get('name'),
          web_property.get('id'),
          custom_metric.get('id'),
          custom_metric.get('name'),
          custom_metric.get('index'),
          custom_metric.get('scope'),
          custom_metric.get('active'),
          custom_metric.get('created'),
          custom_metric.get('updated'),
github google / starthinker / starthinker / task / sdf / run.py View on Github external
def sdf():
  if project.verbose: print('SDF')

  # Download sdf files
  sdf_zip_file = sdf_download(
    project.task['auth'], 
    project.task['version'], 
    project.task['partner_id'], 
    project.task['file_types'], 
    project.task['filter_type'], 
    project.task['read']['filter_ids'])

  # Load data into BigQuery
  sdf_to_bigquery(sdf_zip_file, project.id, project.task['dataset'], project.task['time_partitioned_table'], project.task['create_single_day_table'], project.task.get('table_suffix', ''))
github google / starthinker / starthinker / task / dt / run.py View on Github external
def dt_move_large(dt_file, dt_partition, jobs):
  if project.verbose: print("DT TO TABLE LARGE", dt_partition)

  delimiter = '\n'
  disposition = 'WRITE_TRUNCATE'

  # decompression handler for gzip ( must be outside of chunks as it keeps track of stream across multiple calls )
  gz_handler = zlib.decompressobj(32 + zlib.MAX_WBITS)

  # sliding view of data flowing out of decompression, used to buffer and delimit rows
  first_row = True
  view = ''

  # loop all chunks of file, decompress, and find row delimiter
  for data_gz in object_get_chunks(project.task['auth'], '%s:%s' % (project.task['bucket'], dt_file)):

    view += gz_handler.decompress(data_gz.read()).decode()
github google / starthinker / starthinker / task / ga_settings_download / run.py View on Github external
@project.from_parameters
def ga_settings_download():
  if project.verbose: print('Initiating GA Settings Download')

  filters = [str(f) for f in project.task.get('accounts', [])] # in case integer list is provided
  accounts = [a for a in API_Analytics(project.task['auth'], iterate=True).management().accountSummaries().list().execute() if not filters or a['id'] in filters or a['name'] in filters]

  if accounts:
    current_date = project.date 

    write_to_bigquery(
      'ga_custom_dimension_settings', 
      CUSTOM_DIMENSION_SCHEMA,
      custom_dimensions_download(accounts, current_date),
      'CSV'
    )
github google / starthinker / util / sheets / __init__.py View on Github external
def sheets_read(auth, sheet_url, sheet_tab, sheet_range, retries=10):
  if project.verbose: print 'SHEETS READ', sheet_url, sheet_tab, sheet_range
  service = get_service('sheets', 'v4', auth)
  sheet_id = sheets_id(sheet_url)
  return API_Retry(service.spreadsheets().values().get(spreadsheetId=sheet_id, range=sheets_tab_range(sheet_tab, sheet_range)), 'values', retries=retries)