Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def hdfs_client(self):
url = 'http://{nn_host}:{webhdfs_port}'.format(
nn_host=self._nn_host, webhdfs_port=self._webhdfs_port)
if self._kerberized:
from hdfs.ext.kerberos import KerberosClient
client = KerberosClient(url, mutual_auth='REQUIRED')
else:
from hdfs.client import InsecureClient
client = InsecureClient(url, user=self._hdfs_user)
return client
symbol = function.name
log("Loading types for function %s" % symbol)
# skip the first argument, which is FunctionContext*
arg_types = tuple([llvm2impala[arg.pointee.name]
for arg in function.type.pointee.args[1:]])
functions.append((symbol, arg_types))
except (AttributeError, KeyError):
# this process could fail for non-UDF helper functions...just ignore
# them, because we're not going to be registering them anyway
log("Had trouble with function %s; moving on..." % symbol)
pass
# transfer the LLVM module to HDFS
url = 'http://{nn_host}:{webhdfs_port}'.format(
nn_host=args.nn_host, webhdfs_port=args.webhdfs_port)
hdfs_client = InsecureClient(url, user=args.user)
hdfs_client.write(args.hdfs_path, bc, overwrite=args.force)
log("Transferred LLVM IR to HDFS at %s" % args.hdfs_path)
# register the functions with impala
conn = impala.dbapi.connect(host=args.impala_host, port=args.impala_port)
cursor = conn.cursor(user=args.user)
log("Connected to impalad: %s" % args.impala_host)
if args.db:
cursor.execute('USE %s' % args.db)
cursor.execute("SHOW FUNCTIONS")
registered_functions = cursor.fetchall()
for (udf_name, return_type) in zip(args.name, args.return_type):
log("Registering function %s" % udf_name)
# find matching LLVM symbols to the current UDF name
matches = [pair for pair in functions if udf_name in pair[0]]
if len(matches) == 0:
)
from hdfs.ext.kerberos import KerberosClient
# note SSL
url = '{0}://{1}:{2}'.format(prefix, host, port)
kwds.setdefault('mutual_auth', 'OPTIONAL')
hdfs_client = KerberosClient(url, session=session, **kwds)
else:
if use_https == 'default':
prefix = 'http'
else:
prefix = 'https' if use_https else 'http'
from hdfs.client import InsecureClient
url = '{}://{}:{}'.format(prefix, host, port)
hdfs_client = InsecureClient(url, session=session, **kwds)
return WebHDFS(hdfs_client)
def __init__(self, url, user=None, **kwargs):
user = user or getuser()
session = kwargs.setdefault('session', rq.Session())
if not session.params:
session.params = {}
session.params['user.name'] = user
super(InsecureClient, self).__init__(url, **kwargs)