Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def counting_generator():
try:
for clk in ijson.items(raw_stream, 'clks.item'):
# Often the clients upload base64 strings with newlines
# We remove those here
raw = ''.join(clk.split('\n')).encode() + b'\n'
store['count'] += 1
store['totalbytes'] += len(raw)
yield raw
except ijson.common.IncompleteJSONError as e:
store['count'] = 0
app.logger.warning("Stopping as we have received incomplete json")
return
def import_google(filename):
logging.info('Importing from ' + filename)
# Needs to be rb!
with open(filename, 'rb') as f:
data = ijson.items(f, 'locations.item')
c=0
for o in data:
c+=1
p = (round(o['longitudeE7']/10000000, precision), round(o['latitudeE7']/10000000, precision))
d, t = tst_to_dt(int(o['timestampMs'][:-3]))
make_history(p, d, t, False)
f.close()
logging.info(str(c) + ' items imported from ' + filename)
logging.info('History size: ' + str(len(history)) + ' points')
pickle.dump(history, open('history.pickle', 'wb'))
write_js()
def _get_structure_info(self, view):
"""
"""
#get structure info command
text = view.substr(Region(0, view.size()))
cmd = self.get_structure_info_cmd(view, text)
timeout = self.get_settings(view, "sourcekitten_timeout", 1.0)
# run structure info command
p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
structure_info = list(ijson.items(p.stdout,''))[0]
return structure_info
def getResultsStats(file_name, dest):
import ijson.backends.yajl2_cffi as ijson
log.debug("getResultsStats()")
file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
distributions = []
with open(file_path, 'rb') as results:
i = "1"
while True:
results.seek(0, 0)
tmp = [d for d in ijson.items(results, "reads-distribution-%s.item" % i)]
if len(tmp) == 0:
break
else:
distributions.append((i, tmp[0]))
i += "0"
dest['distribution'] = distributions
return dest
def get_collection_count(source, type):
"""
Helper function to get total collection length.
Input:
- source: str, value denoting where we will read from (e.g 'mongo')
- type: str, value denoting what we will read (e.g. text, edges)
Output:
- N_collection: int,
number of items in the collection
"""
if source == 'file':
inp_path = settings['load']['path']['file_path']
# Document iterator field in the collection
infield = settings['load'][type]['itemfield']
with open(inp_path, 'r') as f:
docs = ijson2.items(f, '%s.item' % infield)
N_collection = 0
for item in docs:
N_collection += 1
elif source == 'mongo':
# input mongo variables from settings.yaml
uri = settings['load']['mongo']['uri']
db_name = settings['load']['mongo']['db']
collection_name = settings['load']['mongo']['collection']
client = pymongo.MongoClient(uri)
db = client[db_name]
collection = db[collection_name]
N_collection = collection.count()
else:
time_log("Can't calculate total collection count for source type %s" % settings['in']['source'])
raise NotImplementedError
return N_collection
def yield_obj(path, basepath):
with gzip.open(path, "r") as fin:
builder = ijson.common.ObjectBuilder()
for prefix, event, val in ijson.parse(fin):
try:
builder.event(event, val)
except:
if hasattr(builder, "value"):
print(builder.value)
if prefix == basepath and event == "end_map":
if hasattr(builder, "value"):
yield builder.value
builder = ijson.common.ObjectBuilder()
def validate(self, filepath):
with open(filepath, 'rb') as vfile:
parser = ijson.parse(vfile)
model = list(self._model_prefixes)
for prefix, event, value in parser:
pair = (prefix, event)
if pair in model:
model.remove(pair)
return len(model) == 0