Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@functools.wraps(fun)
def wrapper(*args, **kwargs):
import luigi.configuration
orig_conf = luigi.configuration.LuigiConfigParser.instance()
new_conf = luigi.configuration.LuigiConfigParser()
luigi.configuration.LuigiConfigParser._instance = new_conf
orig_dict = {k: dict(orig_conf.items(k)) for k in orig_conf.sections()}
new_dict = self._make_dict(orig_dict)
for (section, settings) in six.iteritems(new_dict):
new_conf.add_section(section)
for (name, value) in six.iteritems(settings):
new_conf.set(section, name, value)
try:
return fun(*args, **kwargs)
finally:
luigi.configuration.LuigiConfigParser._instance = orig_conf
return wrapper
def tearDown(self):
os.environ.clear()
os.environ.update(self.environ)
if os.path.exists('client.cfg'):
os.unlink('client.cfg')
luigi.configuration.LuigiConfigParser._instance = None
@functools.wraps(fun)
def wrapper(*args, **kwargs):
import luigi.configuration
orig_conf = luigi.configuration.LuigiConfigParser.instance()
new_conf = luigi.configuration.LuigiConfigParser()
luigi.configuration.LuigiConfigParser._instance = new_conf
orig_dict = {k: dict(orig_conf.items(k)) for k in orig_conf.sections()}
new_dict = self._make_dict(orig_dict)
for (section, settings) in six.iteritems(new_dict):
new_conf.add_section(section)
for (name, value) in six.iteritems(settings):
new_conf.set(section, name, value)
try:
return fun(*args, **kwargs)
finally:
luigi.configuration.LuigiConfigParser._instance = orig_conf
return wrapper
def _config(self, values):
with open('client.cfg', 'w') as f:
f.write("[td]\n")
for key, val in values.items():
f.write("{0}: {1}\n".format(key, val))
luigi.configuration.LuigiConfigParser._instance = None
def load_default(self):
luigi_config = luigi.configuration.get_config()
apikey = luigi_config.get('td', 'apikey', os.environ.get('TD_API_KEY'))
endpoint = luigi_config.get('td', 'endpoint', DEFAULT_ENDPOINT)
self.config = Config(apikey, endpoint=endpoint)
logger = logging.getLogger('luigi-interface')
try:
import mysql.connector
from mysql.connector import errorcode, Error
except ImportError:
logger.warning("Loading MySQL module without the python package mysql-connector-python. \
This will crash at runtime if MySQL functionality is used.")
class MySqlTarget(luigi.Target):
"""
Target for a resource in MySql.
"""
marker_table = luigi.configuration.get_config().get('mysql', 'marker-table', 'table_updates')
def __init__(self, host, database, user, password, table, update_id, **cnx_kwargs):
"""
Initializes a MySqlTarget instance.
:param host: MySql server address. Possibly a host:port string.
:type host: str
:param database: database name.
:type database: str
:param user: database user
:type user: str
:param password: password for specified user.
:type password: str
:param update_id: an identifier for this data set.
:type update_id: str
:param cnx_kwargs: optional params for mysql connector constructor.
def cw_api_sleep():
c = luigi.configuration.get_config()
t = c.getfloat('swfscheduler', 'cw_api_sleep', 0.2)
sleep(t)
import logging
import datetime
import tempfile
import subprocess
from six.moves.urllib.parse import urlparse
from bs4 import BeautifulSoup
import requests
import luigi
from luigi.postgres import PostgresTarget
from .sql_tasks import RunQuery
config = luigi.configuration.get_config()
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('ooni-pipeline')
def download_citizen_lab_test_list():
archive_url = "https://github.com/citizenlab/test-lists/archive/master.zip"
output_directory = tempfile.mkdtemp()
r = requests.get(archive_url, stream=True)
with tempfile.NamedTemporaryFile(delete=False) as fw:
zip_filename = fw.name
for chunk in r.iter_content(chunk_size=1024):
if chunk:
fw.write(chunk)
def get_hive_syntax():
return luigi.configuration.get_config().get('hive', 'release', 'cdh4')
def __init__(self,cfg_name='postgres'):
import psycopg2
self.host = luigi.configuration.get_config().get(cfg_name, 'host')
self.database = luigi.configuration.get_config().get(cfg_name, 'database')
self.user = luigi.configuration.get_config().get(cfg_name, 'user')
self.password = luigi.configuration.get_config().get(cfg_name, 'password')
self.port = luigi.configuration.get_config().get(cfg_name, 'port')
self.connection = self.__connect__()
self.cursor = self.connection.cursor()