Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if args[5] == "-file":
reader = open(args[6], 'rb')
datum_reader = avro.io.DatumReader()
dfr = datafile.DataFileReader(reader, datum_reader)
datum = next(dfr)
elif args[5] == "-data":
print("JSON Decoder not yet implemented.")
return 1
else:
print(usage_str)
return 1
send_message(uri, proto, msg, datum)
return 0
if __name__ == "__main__":
if os.path.dirname(avro.io.__file__) in sys.path:
warnings.warn("Invoking avro/tool.py directly is likely to lead to a name collision with the python io module. Try doing `python -m avro.tool` instead.")
sys.exit(main(sys.argv))
def respond(self, transceiver):
"""Called by a server to deserialize a request, compute and serialize
* a response or error."""
transreq = transceiver.readbuffers()
reader = cStringIO.StringIO(transreq)
decoder = io.Decoder(reader)
buf = cStringIO.StringIO()
encoder = io.Encoder(buf)
error = None
responsemeta = dict()
try:
remoteproto = self.__handshake(transceiver, decoder, encoder)
if remoteproto is None: #handshake failed
return buf.getvalue()
#read request using remote protocol specification
requestmeta = _META_READER.read(decoder)
msgname = decoder.readutf8()
m = remoteproto.getmessages().get(msgname)
if m is None:
raise schema.AvroException("No such remote message: "+msgname.__str__())
data = self.raw_decoder.read(length - 4)
uncompressed = snappy.decompress(data)
self._datum_decoder = avro.io.BinaryDecoder(io.BytesIO(uncompressed))
self.raw_decoder.check_crc32(uncompressed);
elif self.codec == 'zstandard':
length = self.raw_decoder.read_long()
data = self.raw_decoder.read(length)
uncompressed = bytearray()
dctx = zstd.ZstdDecompressor()
with dctx.stream_reader(io.BytesIO(data)) as reader:
while True:
chunk = reader.read(16384)
if not chunk:
break
uncompressed.extend(chunk)
self._datum_decoder = avro.io.BinaryDecoder(io.BytesIO(uncompressed))
else:
raise DataFileException("Unknown codec: %r" % self.codec)
def avro_writer(self):
return avro.io.DatumWriter(
writers_schema=self.schema
)
if not avro_schema_file:
reader = DataFileReader(open(avro_file, "r"), DatumReader())
for datum in reader:
print datum
reader.close()
else:
reader_schema = open(avro_schema_file, "r")
avro_schema = reader_schema.read()
reader_schema.close()
parsed_avro_schema = avro.schema.parse(avro_schema)
with open(avro_file, "rb") as reader_data:
inputio = io.BytesIO(reader_data.read())
decoder = avro.io.BinaryDecoder(inputio)
reader = avro.io.DatumReader(parsed_avro_schema)
while inputio.tell() < len(inputio.getvalue()):
avro_datum = reader.read(decoder)
print avro_datum
reader_data.close()
schema.INT : lambda schm, pkgname, object: ((isinstance(object, long) or
isinstance(object, int)) and
io._INT_MIN_VALUE <= object <= io._INT_MAX_VALUE),
schema.LONG : lambda schm, pkgname, object: ((isinstance(object, long) or
:param schema.Schema writers_schema:
:param schema.Schema readers_schema:
:param io.BinaryDecoder decoder:
:return:
"""
result = super(LogicalDatumReader, self).read_data(writers_schema, readers_schema, decoder)
logical_type = readers_schema.props.get('logicalType')
if logical_type:
logical_type_handler = self.logical_types.get(logical_type)
if logical_type_handler and logical_type_handler.does_match(writers_schema, readers_schema):
result = logical_type_handler.convert_back(writers_schema, readers_schema, result)
return result
class LogicalDatumWriter(io.DatumWriter):
"""
Initializes DatumWriter with logical type support
:param schema.Schema writers_schema: Writer's schema
:param dict[str, LogicalTypeProcessor] logical_types: Optional logical types dict
"""
def __init__(self, writers_schema=None, logical_types=DEFAULT_LOGICAL_TYPES):
super(LogicalDatumWriter, self).__init__(writers_schema=writers_schema)
self.logical_types = logical_types
def write_data(self, writers_schema, datum, encoder):
logical_type = writers_schema.props.get('logicalType')
if logical_type:
logical_type_handler = self.logical_types.get(logical_type)
if logical_type_handler and logical_type_handler.can_convert(writers_schema):