How to use the petl.compat.pickle.dump function in petl

To help you get started, we’ve selected a few petl examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github petl-developers / petl / src / petl / io / pickle.py View on Github external
def _writepickle(table, source, mode, protocol, write_header):
    source = write_source_from_arg(source)
    with source.open_(mode) as f:
        it = iter(table)
        hdr = next(it)
        if write_header:
            pickle.dump(hdr, f, protocol)
        for row in it:
            pickle.dump(row, f, protocol)
github petl-developers / petl / petl / push.py View on Github external
def accept(self, row):
        pickle.dump(row, self.file, self.protocol)
        # forward rows on the default pipe (behave like tee)
        self.broadcast(row)
github petl-developers / petl / petl / io / pickle.py View on Github external
def _writepickle(table, source, mode, protocol, write_header):
    source = write_source_from_arg(source)
    with source.open(mode) as f:
        it = iter(table)
        hdr = next(it)
        if write_header:
            pickle.dump(hdr, f, protocol)
        for row in it:
            pickle.dump(row, f, protocol)
github petl-developers / petl / src / petl / transform / sorts.py View on Github external
# actually not needed to iterate from memcache
                self._getkey = getkey

            for row in rows:
                yield tuple(row)

        else:

            chunkfiles = []

            while rows:

                # dump the chunk
                f = NamedTemporaryFile(dir=self.tempdir)
                for row in rows:
                    pickle.dump(row, f, protocol=-1)
                f.flush()
                # N.B., do not close the file! Closing will delete
                # the file, and we might want to keep it around
                # if it can be cached. We'll let garbage collection
                # deal with this, i.e., when no references to the
                # chunk files exist any more, garbage collection
                # should be an implicit close, which will cause file
                # deletion.
                chunkfiles.append(f)

                # grab the next chunk
                rows = list(itertools.islice(it, 0, self.buffersize))
                rows.sort(key=getkey, reverse=reverse)

            if self.cache:
                debug('caching files %r', chunkfiles)
github petl-developers / petl / petl / io / pickle.py View on Github external
def _writepickle(table, source, mode, protocol, write_header):
    source = write_source_from_arg(source)
    with source.open(mode) as f:
        it = iter(table)
        hdr = next(it)
        if write_header:
            pickle.dump(hdr, f, protocol)
        for row in it:
            pickle.dump(row, f, protocol)
github petl-developers / petl / src / petl / push.py View on Github external
def __init__(self, default_connections, keyed_connections, fields,
                 filename, protocol):
        super(ToPickleConnection, self).__init__(default_connections,
                                                 keyed_connections, fields)
        self.file = open(filename, 'wb')
        self.protocol = protocol
        pickle.dump(fields, self.file, self.protocol)
github petl-developers / petl / src / petl / io / pickle.py View on Github external
def _writepickle(table, source, mode, protocol, write_header):
    source = write_source_from_arg(source)
    with source.open_(mode) as f:
        it = iter(table)
        hdr = next(it)
        if write_header:
            pickle.dump(hdr, f, protocol)
        for row in it:
            pickle.dump(row, f, protocol)