How to use the avro.io function in avro

To help you get started, we’ve selected a few avro examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github apache / avro / lang / py / avro / tool.py View on Github external
if args[5] == "-file":
        reader = open(args[6], 'rb')
        datum_reader = avro.io.DatumReader()
        dfr = datafile.DataFileReader(reader, datum_reader)
        datum = next(dfr)
      elif args[5] == "-data":
        print("JSON Decoder not yet implemented.")
        return 1
      else:
        print(usage_str)
        return 1
    send_message(uri, proto, msg, datum)
  return 0

if __name__ == "__main__":
  if os.path.dirname(avro.io.__file__) in sys.path:
    warnings.warn("Invoking avro/tool.py directly is likely to lead to a name collision with the python io module. Try doing `python -m avro.tool` instead.")

  sys.exit(main(sys.argv))
github johnj / php5-xcom / src / py / avro / ipc.py View on Github external
def respond(self, transceiver):
    """Called by a server to deserialize a request, compute and serialize
   * a response or error."""
    transreq = transceiver.readbuffers()
    reader = cStringIO.StringIO(transreq)
    decoder = io.Decoder(reader)
    buf = cStringIO.StringIO()
    encoder = io.Encoder(buf)
    error = None
    responsemeta = dict()
    
    try:
      remoteproto = self.__handshake(transceiver, decoder, encoder)
      if remoteproto is None:  #handshake failed
        return buf.getvalue()
      
      #read request using remote protocol specification
      requestmeta = _META_READER.read(decoder)
      msgname = decoder.readutf8()
      m = remoteproto.getmessages().get(msgname)
      if m is None:
        raise schema.AvroException("No such remote message: "+msgname.__str__())
github apache / avro / lang / py / avro / datafile.py View on Github external
data = self.raw_decoder.read(length - 4)
      uncompressed = snappy.decompress(data)
      self._datum_decoder = avro.io.BinaryDecoder(io.BytesIO(uncompressed))
      self.raw_decoder.check_crc32(uncompressed);
    elif self.codec == 'zstandard':
      length = self.raw_decoder.read_long()
      data = self.raw_decoder.read(length)
      uncompressed = bytearray()
      dctx = zstd.ZstdDecompressor()
      with dctx.stream_reader(io.BytesIO(data)) as reader:
        while True:
          chunk = reader.read(16384)
          if not chunk:
            break
          uncompressed.extend(chunk)
      self._datum_decoder = avro.io.BinaryDecoder(io.BytesIO(uncompressed))
    else:
      raise DataFileException("Unknown codec: %r" % self.codec)
github Yelp / data_pipeline_avro_util / data_pipeline_avro_util / avro_string_writer.py View on Github external
def avro_writer(self):
        return avro.io.DatumWriter(
            writers_schema=self.schema
        )
github pmacct / pmacct / examples / avro / avro_file_decoder.py View on Github external
if not avro_schema_file:
		reader = DataFileReader(open(avro_file, "r"), DatumReader())
		for datum in reader:
			print datum
		reader.close()
	else:
		reader_schema = open(avro_schema_file, "r")
		avro_schema = reader_schema.read()
		reader_schema.close()
		parsed_avro_schema = avro.schema.parse(avro_schema)

		with open(avro_file, "rb") as reader_data:
			inputio = io.BytesIO(reader_data.read())
			decoder = avro.io.BinaryDecoder(inputio)
			reader = avro.io.DatumReader(parsed_avro_schema)
			while inputio.tell() < len(inputio.getvalue()):
				avro_datum = reader.read(decoder)
				print avro_datum
		reader_data.close()
github johnj / php5-xcom / src / py / avro / reflectio.py View on Github external
schema.INT : lambda schm, pkgname, object: ((isinstance(object, long) or 
                                         isinstance(object, int)) and 
                              io._INT_MIN_VALUE <= object <= io._INT_MAX_VALUE),
     schema.LONG : lambda schm, pkgname, object: ((isinstance(object, long) or
github rbystrit / avro_gen / avrogen / logical.py View on Github external
:param schema.Schema writers_schema:
        :param schema.Schema readers_schema:
        :param io.BinaryDecoder decoder:
        :return:
        """
        result = super(LogicalDatumReader, self).read_data(writers_schema, readers_schema, decoder)
        logical_type = readers_schema.props.get('logicalType')
        if logical_type:
            logical_type_handler = self.logical_types.get(logical_type)
            if logical_type_handler and logical_type_handler.does_match(writers_schema, readers_schema):
                result = logical_type_handler.convert_back(writers_schema, readers_schema, result)
        return result


class LogicalDatumWriter(io.DatumWriter):
    """
       Initializes DatumWriter with logical type support

       :param schema.Schema writers_schema: Writer's schema
       :param dict[str, LogicalTypeProcessor] logical_types: Optional logical types dict
       """

    def __init__(self, writers_schema=None, logical_types=DEFAULT_LOGICAL_TYPES):
        super(LogicalDatumWriter, self).__init__(writers_schema=writers_schema)
        self.logical_types = logical_types

    def write_data(self, writers_schema, datum, encoder):
        logical_type = writers_schema.props.get('logicalType')
        if logical_type:
            logical_type_handler = self.logical_types.get(logical_type)
            if logical_type_handler and logical_type_handler.can_convert(writers_schema):