Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_parse_float(self):
eq_(parse_arg({'foo': '123.4'}, 'foo', float), 123.4)
def test_parse_invalid(self):
parse_arg({'foo': 'a'}, 'foo', int)
def test_parse_int(self):
eq_(parse_arg({'foo': '1'}, 'foo', int), 1)
eq_(parse_arg({'foo': '1'}, 'foo', int, ','), 1)
def test_parse_int_list(self):
eq_(parse_arg({'foo': '1,'}, 'foo', int, ','), [1])
eq_(parse_arg({'foo': '1,2'}, 'foo', int, ','), [1,2])
def test_parse_int(self):
eq_(parse_arg({'foo': '1'}, 'foo', int), 1)
eq_(parse_arg({'foo': '1'}, 'foo', int, ','), 1)
overwrite=overwrite,
schema=parse_arg(args, '--schema', loads),
codec=args['--codec'],
)
with writer:
records = (loads(line) for line in stdin)
for record in records:
writer.write(record)
else:
reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
with reader:
if args['schema']:
stdout.write('%s\n' % (dumps(reader.schema, indent=2), ))
elif args['read']:
encoder = _Encoder()
num = parse_arg(args, '--num', int)
freq = parse_arg(args, '--freq', float)
if freq:
for record in reader:
if random() <= freq:
stdout.write(encoder.encode(record))
stdout.write('\n')
else:
for record in islice(reader, num):
stdout.write(encoder.encode(record))
stdout.write('\n')
def main(argv=None, client=None, stdin=sys.stdin, stdout=sys.stdout):
"""Entry point.
:param argv: Arguments list.
:param client: For testing.
"""
args = docopt(__doc__, argv=argv)
if not client:
client = configure_client('hdfscli-avro', args)
elif args['--log']:
raise HdfsError('Logging is only available when no client is specified.')
overwrite = args['--force']
parts = parse_arg(args, '--parts', int, ',')
if args['write']:
writer = AvroWriter(
client,
args['HDFS_PATH'],
overwrite=overwrite,
schema=parse_arg(args, '--schema', loads),
codec=args['--codec'],
)
with writer:
records = (loads(line) for line in stdin)
for record in records:
writer.write(record)
else:
reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
with reader:
if args['schema']:
schema=parse_arg(args, '--schema', loads),
codec=args['--codec'],
)
with writer:
records = (loads(line) for line in stdin)
for record in records:
writer.write(record)
else:
reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
with reader:
if args['schema']:
stdout.write('%s\n' % (dumps(reader.schema, indent=2), ))
elif args['read']:
encoder = _Encoder()
num = parse_arg(args, '--num', int)
freq = parse_arg(args, '--freq', float)
if freq:
for record in reader:
if random() <= freq:
stdout.write(encoder.encode(record))
stdout.write('\n')
else:
for record in islice(reader, num):
stdout.write(encoder.encode(record))
stdout.write('\n')
:param client: For testing.
"""
args = docopt(__doc__, argv=argv)
if not client:
client = configure_client('hdfscli-avro', args)
elif args['--log']:
raise HdfsError('Logging is only available when no client is specified.')
overwrite = args['--force']
parts = parse_arg(args, '--parts', int, ',')
if args['write']:
writer = AvroWriter(
client,
args['HDFS_PATH'],
overwrite=overwrite,
schema=parse_arg(args, '--schema', loads),
codec=args['--codec'],
)
with writer:
records = (loads(line) for line in stdin)
for record in records:
writer.write(record)
else:
reader = AvroReader(client, args['HDFS_PATH'], parts=parts)
with reader:
if args['schema']:
stdout.write('%s\n' % (dumps(reader.schema, indent=2), ))
elif args['read']:
encoder = _Encoder()
num = parse_arg(args, '--num', int)
freq = parse_arg(args, '--freq', float)
if freq: