Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _writepickle(table, source, mode, protocol, write_header):
source = write_source_from_arg(source)
with source.open_(mode) as f:
it = iter(table)
hdr = next(it)
if write_header:
pickle.dump(hdr, f, protocol)
for row in it:
pickle.dump(row, f, protocol)
def accept(self, row):
pickle.dump(row, self.file, self.protocol)
# forward rows on the default pipe (behave like tee)
self.broadcast(row)
def _writepickle(table, source, mode, protocol, write_header):
source = write_source_from_arg(source)
with source.open(mode) as f:
it = iter(table)
hdr = next(it)
if write_header:
pickle.dump(hdr, f, protocol)
for row in it:
pickle.dump(row, f, protocol)
# actually not needed to iterate from memcache
self._getkey = getkey
for row in rows:
yield tuple(row)
else:
chunkfiles = []
while rows:
# dump the chunk
f = NamedTemporaryFile(dir=self.tempdir)
for row in rows:
pickle.dump(row, f, protocol=-1)
f.flush()
# N.B., do not close the file! Closing will delete
# the file, and we might want to keep it around
# if it can be cached. We'll let garbage collection
# deal with this, i.e., when no references to the
# chunk files exist any more, garbage collection
# should be an implicit close, which will cause file
# deletion.
chunkfiles.append(f)
# grab the next chunk
rows = list(itertools.islice(it, 0, self.buffersize))
rows.sort(key=getkey, reverse=reverse)
if self.cache:
debug('caching files %r', chunkfiles)
def _writepickle(table, source, mode, protocol, write_header):
source = write_source_from_arg(source)
with source.open(mode) as f:
it = iter(table)
hdr = next(it)
if write_header:
pickle.dump(hdr, f, protocol)
for row in it:
pickle.dump(row, f, protocol)
def __init__(self, default_connections, keyed_connections, fields,
filename, protocol):
super(ToPickleConnection, self).__init__(default_connections,
keyed_connections, fields)
self.file = open(filename, 'wb')
self.protocol = protocol
pickle.dump(fields, self.file, self.protocol)
def _writepickle(table, source, mode, protocol, write_header):
source = write_source_from_arg(source)
with source.open_(mode) as f:
it = iter(table)
hdr = next(it)
if write_header:
pickle.dump(hdr, f, protocol)
for row in it:
pickle.dump(row, f, protocol)