How to use the srsly.cloudpickle function in srsly

To help you get started, we’ve selected a few srsly examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github explosion / thinc / examples / cnn_tagger.py View on Github external
trainer.each_epoch.append(track_progress(**locals()))
        trainer.batch_size = min_batch_size
        batch_size = float(min_batch_size)
        for X, y in trainer.iterate(train_X, train_y):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)

            gradient = [yh[i] - y[i] for i in range(len(yh))]

            backprop(gradient, optimizer)

            trainer.batch_size = min(int(batch_size), max_batch_size)
            batch_size *= 1.001
    with model.use_params(trainer.optimizer.averages):
        print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
        with open("/tmp/model.pickle", "wb") as file_:
            pickle.dump(model, file_)
github explosion / thinc / examples / ngram_bow.py View on Github external
epoch_loss.append(0.0)

        trainer.each_epoch.append(report_progress)
        trainer.nb_epoch = nb_epoch
        trainer.dropout = 0.0
        trainer.batch_size = 128
        trainer.dropout_decay = 0.0
        for X, y in trainer.iterate(train_X[:1000], train_y[:1000]):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)
            loss = ((yh - y) ** 2.0).sum() / y.shape[0]
            backprop((yh - y) / y.shape[0], optimizer)
            epoch_loss[-1] += loss
        with model.use_params(optimizer.averages):
            print("Avg dev.: %.3f" % model.evaluate(dev_X, dev_y))
            with open("out.pickle", "wb") as file_:
                pickle.dump(model, file_, -1)
github explosion / thinc / examples / text-pair / glove_mwe_multipool_siamese.py View on Github external
assert (yh >= 0.0).all(), yh
            train_acc = ((yh >= 0.5) == (y >= 0.5)).sum()
            loss = model.ops.xp.abs(yh - y).mean()
            epoch_train_acc += train_acc
            backprop(yh - y, optimizer)
            n_iter += 1

            # Slightly useful trick: start with low batch size, accelerate.
            trainer.batch_size = min(int(batch_size), max_batch_size)
            batch_size *= 1.001
        if out_loc:
            out_loc = Path(out_loc)
            print("Saving to", out_loc)
            with out_loc.open("wb") as file_:
                pickle.dump(model, file_, -1)
github explosion / thinc / examples / pytorch_lstm_tagger.py View on Github external
):
        trainer.each_epoch.append(track_progress(**locals()))
        trainer.batch_size = min_batch_size
        batch_size = float(min_batch_size)
        for X, y in trainer.iterate(train_X, train_y):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)

            gradient = [yh[i] - y[i] for i in range(len(yh))]

            backprop(gradient, optimizer)

            trainer.batch_size = min(int(batch_size), max_batch_size)
            batch_size *= 1.001
    print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
    with open("/tmp/model.pickle", "wb") as file_:
        pickle.dump(model, file_)
github explosion / thinc / examples / lstm_tagger.py View on Github external
):
        trainer.each_epoch.append(track_progress(**locals()))
        trainer.batch_size = min_batch_size
        batch_size = float(min_batch_size)
        for X, y in trainer.iterate(train_X, train_y):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)

            gradient = [yh[i] - y[i] for i in range(len(yh))]

            backprop(gradient, optimizer)

            trainer.batch_size = min(int(batch_size), max_batch_size)
            batch_size *= 1.001
    print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
    with open("/tmp/model.pickle", "wb") as file_:
        pickle.dump(model, file_)
github explosion / thinc / examples / mnist_mlp.py View on Github external
trainer.each_epoch.append(report_progress)
        trainer.nb_epoch = nb_epoch
        trainer.dropout = 0.3
        trainer.batch_size = 128
        trainer.dropout_decay = 0.0
        train_X = model.ops.asarray(train_X, dtype="float32")
        y_onehot = to_categorical(train_y)
        for X, y in trainer.iterate(train_X, y_onehot):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)
            loss = ((yh - y) ** 2.0).sum() / y.shape[0]
            backprop(yh - y, optimizer)
            epoch_loss[-1] += loss
        with model.use_params(optimizer.averages):
            print("Avg dev.: %.3f" % model.evaluate(dev_X, dev_y))
            with open("out.pickle", "wb") as file_:
                pickle.dump(model, file_, -1)
github explosion / thinc / thinc / extra / datasets.py View on Github external
def get_word_index(path="reuters_word_index.pkl"):  # pragma: no cover
    path = get_file(
        path, origin="https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl"
    )
    f = open(path, "rb")

    if sys.version_info < (3,):
        data = pickle.load(f)
    else:
        data = pickle.load(f, encoding="latin1")

    f.close()
    return data
github explosion / srsly / srsly / _pickle_api.py View on Github external
def pickle_loads(data):
    """Deserialize bytes with pickle.

    data (bytes): The data to deserialize.
    RETURNS: The deserialized Python object.
    """
    return cloudpickle.loads(data)
github explosion / thinc / thinc / extra / datasets.py View on Github external
def get_word_index(path="reuters_word_index.pkl"):  # pragma: no cover
    path = get_file(
        path, origin="https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl"
    )
    f = open(path, "rb")

    if sys.version_info < (3,):
        data = pickle.load(f)
    else:
        data = pickle.load(f, encoding="latin1")

    f.close()
    return data
github explosion / srsly / srsly / _pickle_api.py View on Github external
def pickle_dumps(data, protocol=None):
    """Serialize a Python object with pickle.

    data: The object to serialize.
    protocol (int): Protocol to use. -1 for highest.
    RETURNS (bytest): The serialized object.
    """
    return cloudpickle.dumps(data, protocol=protocol)