How to use the petl.util.close function in petl

To help you get started, we’ve selected a few petl examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github petl-developers / petl / src / petl / transform.py View on Github external
except StopIteration:
            pass
        
        # any left over?
        if lkval > rkval:
            # yield anything that got left hanging
            for row in lrowgrp:
                yield row
        # and the rest...
        for lkval, lrowgrp in lgit:
            for row in lrowgrp:
                yield row

    finally:
        close(lit)
        close(rit)
github petl-developers / petl / src / petl / transform.py View on Github external
if field in flds:
            index = flds.index(field)
        elif isinstance(field, int) and field < len(flds):
            index = field
        else:
            raise FieldSelectionError(field)
        
        for row in it:
            row = list(row) # copy, so we don't modify the source
            value = row[index]
            if value in dictionary:
                row[index] = dictionary[value]
            yield row
            
    finally:
        close(it)
github petl-developers / petl / src / petl / transform.py View on Github external
variable_indices = [fields.index(f) for f in variable_fields]
        
        # determine the actual variable names to be cast as fields
        if isinstance(variable_fields, dict):
            # user supplied dictionary
            variables = variable_fields
        else:
            variables = defaultdict(set)
            # sample the data to discover variables to be cast as fields
            for row in islice(it, 0, sample_size):
                for i, f in zip(variable_indices, variable_fields):
                    variables[f].add(row[i])
            for f in variables:
                variables[f] = sorted(variables[f]) # turn from sets to sorted lists
        
        close(it) # finished the first pass
        
        # determine the output fields
        out_fields = list(key_fields)
        for f in variable_fields:
            out_fields.extend(variables[f])
        yield out_fields
        
        # output data
        
        source = sort(source, key=key_fields)
        it = iter(source)
        it = islice(it, 1, None) # skip header row
        getkey = itemgetter(*key_indices)
        
        # process sorted data in newfields
        groups = groupby(it, key=getkey)
github petl-developers / petl / src / petl / transform.py View on Github external
values = [r[value_index] for r in group if r[i] == variable]
                    if len(values) == 0:
                        value = missing
                    elif len(values) == 1:
                        value = values[0]
                    else:
                        if variable in reducers:
                            redu = reducers[variable]
                        else:
                            redu = list # list all values
                        value = redu(values)
                    out_row.append(value)
            yield out_row
                    
    finally:
        close(it)
github petl-developers / petl / src / petl / transform.py View on Github external
def iterrowmap(source, rowmapper, header, failonerror):
    it = iter(source)
    try:
        it.next() # discard source header
        yield header
        for row in it:
            try:
                outrow = rowmapper(row)
                yield outrow
            except:
                if failonerror:
                    raise
    finally:
        close(it)
github petl-developers / petl / src / petl / transform.py View on Github external
def iterextendfields(source, flds):
    it = iter(source)
    try:
        srcflds = it.next() 
        outflds = list(srcflds)
        outflds.extend(flds)
        yield outflds
        for row in it:
            yield row
    finally:
        close(it)
github petl-developers / petl / src / petl / transform.py View on Github external
a = ita.next()
                except StopIteration:
                    break
            elif a == b:
                try:
                    a = ita.next()
                except StopIteration:
                    break
            else:
                try:
                    b = itb.next()
                except StopIteration:
                    b = None
    finally:
        close(ita)
        close(itb)
github petl-developers / petl / src / petl / transform.py View on Github external
value = row[flds.index(f)]
                except ValueError: # source does not have f in flds
                    value = missing
                except IndexError: # row is short
                    value = missing
                return value
            
            # now construct and yield the data rows
            for row in it:
                out_row = [get_value(row, f) for f in out_flds]
                yield out_row

    finally:
        # make sure all iterators are closed
        for it in its:
            close(it)
github petl-developers / petl / src / petl / transform.py View on Github external
yield header

        # convert field selection into field indices
        indices = asindices(srcflds, key)
        
        # now use field indices to construct a getkey function
        # N.B., this may raise an exception on short rows, depending on
        # the field selection
        getkey = itemgetter(*indices)
        
        for key, rows in groupby(it, key=getkey):
            records = [asdict(srcflds, row) for row in rows]
            yield reducer(key, records)
        
    finally:
        close(it)
github petl-developers / petl / src / petl / transform.py View on Github external
def iterpushfields(source, flds):
    it = iter(source)
    try:
        yield flds
        for row in it:
            yield row
    finally:
        close(it)