Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# The build process should copy InputProtocol.avpr and OutputProtocol.avpr
# into the same directory as this module
inputProtocol=None
outputProtocol=None
TaskType=None
if (inputProtocol is None):
pfile=os.path.split(__file__)[0]+os.sep+"InputProtocol.avpr"
if not(os.path.exists(pfile)):
raise Exception("Could not locate the InputProtocol: {0} does not exist".format(pfile))
with file(pfile,'r') as hf:
prototxt=hf.read()
inputProtocol=protocol.parse(prototxt)
# use a named tuple to represent the tasktype enumeration
taskschema=inputProtocol.types_dict["TaskType"]
_ttype=collections.namedtuple("_tasktype",taskschema.symbols)
TaskType=_ttype(*taskschema.symbols)
if (outputProtocol is None):
pfile=os.path.split(__file__)[0]+os.sep+"OutputProtocol.avpr"
if not(os.path.exists(pfile)):
raise Exception("Could not locate the OutputProtocol: {0} does not exist".format(pfile))
with file(pfile,'r') as hf:
prototxt=hf.read()
outputProtocol=protocol.parse(prototxt)
def get_client(host='127.0.0.1', port=9170):
schema = os.path.join(root, 'interface/avro', 'cassandra.avpr')
proto = protocol.parse(open(schema).read())
client = ipc.HTTPTransceiver(host, port)
return ipc.Requestor(proto, client)
for the response from the last layer.
"""
import time
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from SocketServer import ThreadingMixIn
from multiprocessing import Queue
from threading import Thread
import avro.ipc as ipc
import avro.protocol as protocol
import avro.schema as schema
import numpy as np
import yaml
# data packet format definition
PROTOCOL = protocol.parse(open('resource/image.avpr').read())
class Initializer:
"""
Singleton factory for initializer. The Initializer module has two timers.
The node_timer is for recording statistics for block1 layer model inference
time. The timer is for recording the total inference time from last
fully connected layer.
Attributes:
queue: Queue for storing available block1 models devices.
start: Start time of getting a frame.
count: Total Number of frames gets back.
node_total: Total layer-wise time.
node_count: Total layer-wise frame count.
"""
match = handshake_response.get('match')
if match == 'BOTH':
self.send_protocol = False
return True
elif match == 'CLIENT':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = False
return True
elif match == 'NONE':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = True
return False
else:
raise schema.AvroException('Unexpected match: %s' % match)
def process_handshake(self, decoder, encoder):
handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
handshake_response = {}
# determine the remote protocol
client_hash = handshake_request.get('clientHash')
client_protocol = handshake_request.get('clientProtocol')
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
server_hash = handshake_request.get('serverHash')
if self.local_hash == server_hash:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'BOTH'
else:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'CLIENT'
if handshake_response['match'] != 'BOTH':
import os
import sys
import avro.ipc as ipc
import avro.protocol as protocol
# TODO(hammer): Figure the canonical place to put this file
PROTO_FILE = os.path.join(os.path.dirname(__file__), 'schema/hbase.avpr')
PROTOCOL = protocol.parse(open(PROTO_FILE).read())
def retry_wrapper(fn):
"""a decorator to add retry symantics to any method that uses hbase"""
def f(self, *args, **kwargs):
try:
return fn(self, *args, **kwargs)
except:
try:
self.close()
except:
pass
self.make_connection()
return fn(self, *args, **kwargs)
return f
class HBaseConnection(object):
def get_client(host='127.0.0.1', port=9170):
schema = os.path.join(root, 'interface/avro', 'cassandra.avpr')
proto = protocol.parse(open(schema).read())
client = ipc.HTTPTransceiver(host, port)
return ipc.Requestor(proto, client)