Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def build_simple_cnn_text_classifier(tok2vec, nr_class, exclusive_classes=False, **cfg):
"""
Build a simple CNN text classifier, given a token-to-vector model as inputs.
If exclusive_classes=True, a softmax non-linearity is applied, so that the
outputs sum to 1. If exclusive_classes=False, a logistic non-linearity
is applied instead, so that outputs are in the range [0, 1].
"""
with Model.define_operators({">>": chain}):
if exclusive_classes:
output_layer = Softmax(nr_class, tok2vec.nO)
else:
output_layer = (
zero_init(Affine(nr_class, tok2vec.nO, drop_factor=0.0)) >> logistic
)
model = tok2vec >> flatten_add_lengths >> Pooling(mean_pool) >> output_layer
model.tok2vec = chain(tok2vec, flatten)
model.nO = nr_class
return model
def main(depth=2, width=512, nb_epoch=30):
prefer_gpu()
# Configuration here isn't especially good. But, for demo..
with Model.define_operators({"**": clone, ">>": chain}):
model = ReLu(width) >> ReLu(width) >> Softmax()
train_data, dev_data, _ = datasets.mnist()
train_X, train_y = model.ops.unzip(train_data)
dev_X, dev_y = model.ops.unzip(dev_data)
dev_y = to_categorical(dev_y)
with model.begin_training(train_X, train_y, L2=1e-6) as (trainer, optimizer):
epoch_loss = [0.0]
def report_progress():
with model.use_params(optimizer.averages):
print(epoch_loss[-1], model.evaluate(dev_X, dev_y), trainer.dropout)
epoch_loss.append(0.0)
trainer.each_epoch.append(report_progress)
trainer.nb_epoch = nb_epoch
def softmax_class_vector(nr_class, *, exclusive_classes=True, **cfg):
"""Select features from the class-vectors from the last hidden state,
mean-pool them, and apply a softmax-activated hidden layer to produce one
vector per document. The gradients of the class vectors are incremented
in the backward pass, to allow fine-tuning.
"""
width = cfg["token_vector_width"]
return chain(
get_class_tokens,
flatten_add_lengths,
Pooling(mean_pool),
Softmax(nr_class, width),
)
>> Residual(LN(Maxout(width))) ** depth
)
model = (
foreach(sent2vec, drop_factor=2.0)
>> flatten_add_lengths
# This block would allow the model to learn some cross-sentence
# features. It's not useful on this problem. It might make more
# sense to use a BiLSTM here, following Liang et al (2016).
# >> with_getitem(0,
# Residual(ExtractWindow(nW=1) >> LN(Maxout(width))) ** conv_depth
# )
>> ParametricAttention(width, hard=False)
>> Pooling(sum_pool)
>> Residual(LN(Maxout(width))) ** depth
>> Softmax(nr_class)
)
model.lsuv = False
return model
>> Pooling(mean_pool)
>> Residual(LN(Maxout(width)))
)
model = (
foreach(sent2vec, drop_factor=2.0)
>> Residual(
prepare_self_attention(Affine(width*3, width), nM=width, nH=4)
>> MultiHeadedAttention()
>> with_flatten(LN(Affine(width, width)))
)
>> flatten_add_lengths
>> ParametricAttention(width, hard=False)
>> Pooling(mean_pool)
>> Residual(LN(Maxout(width))) ** 2
>> Softmax(nr_class)
)
model.lsuv = False
return model
def main(width=32, nr_vector=1000):
train_data, check_data, nr_tag = ancora_pos_tags(encode_words=True)
model = with_flatten(
chain(
HashEmbed(width, nr_vector),
ReLu(width, width),
ReLu(width, width),
Softmax(nr_tag, width),
)
)
train_X, train_y = zip(*train_data)
dev_X, dev_y = zip(*check_data)
train_y = [to_categorical(y, nb_classes=nr_tag) for y in train_y]
dev_y = [to_categorical(y, nb_classes=nr_tag) for y in dev_y]
with model.begin_training(train_X, train_y) as (trainer, optimizer):
trainer.each_epoch.append(lambda: print(model.evaluate(dev_X, dev_y)))
for X, y in trainer.iterate(train_X, train_y):
yh, backprop = model.begin_update(X, drop=trainer.dropout)
backprop([yh[i] - y[i] for i in range(len(yh))], optimizer)
with model.use_params(optimizer.averages):
print(model.evaluate(dev_X, dev_y))
def build_textcat_model(tok2vec, nr_class, width):
from thinc.v2v import Model, Softmax
from thinc.api import flatten_add_lengths, chain
from thinc.t2v import Pooling, mean_pool
with Model.define_operators({">>": chain}):
model = (
tok2vec
>> flatten_add_lengths
>> Pooling(mean_pool)
>> Softmax(nr_class, width)
)
model.tok2vec = chain(tok2vec, flatten)
return model
def build_model(nr_class, width, **kwargs):
with Model.define_operators({"|": concatenate, ">>": chain, "**": clone}):
model = (
FeatureExtracter([ORTH])
>> flatten_add_lengths
>> with_getitem(0, uniqued(HashEmbed(width, 10000, column=0)))
>> Pooling(mean_pool)
>> Softmax(nr_class)
)
model.lsuv = False
return model