Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
_fit_batches(model, iterator, train_config)
elif callable(getattr(model, 'fit', None)):
_fit(model, iterator, train_config)
elif not isinstance(model, Chainer):
log.warning('Nothing to train')
model.destroy()
res = {}
if train_config['validate_best'] or train_config['test_best']:
# try:
# model_config['load_path'] = model_config['save_path']
# except KeyError:
# log.warning('No "save_path" parameter for the model, so "load_path" will not be renewed')
model = build_model_from_config(config, load_trained=True)
log.info('Testing the best saved model')
if train_config['validate_best']:
report = {
'valid': _test_model(model, metrics_functions, iterator,
train_config.get('batch_size', -1), 'valid',
show_examples=train_config['show_examples'])
}
res['valid'] = report['valid']['metrics']
print(json.dumps(report, ensure_ascii=False))
if train_config['test_best']:
report = {
'test': _test_model(model, metrics_functions, iterator,
def main():
args = parser.parse_args()
config = read_json(args.config_path)
ranker = build_model_from_config(config) # chainer
dataset = read_csv(args.dataset_path)
output_path = args.output_path
with open(output_path, 'w') as fout:
result = []
i = 0
for instance in dataset:
print('Processing {} query'.format(i))
q = instance['question']
a = instance['answer']
pred = ranker([q])
result.append({'question': q, 'answer': a, 'context': pred})
i += 1
json.dump(result, fout)
def main():
args = parser.parse_args()
config = read_json(args.config_path)
ranker = build_model_from_config(config) # chainer
dataset = read_json(args.dataset_path)
# dataset = dataset[:10]
qa_dataset_size = len(dataset)
logger.info('QA dataset size: {}'.format(qa_dataset_size))
# n_queries = 0 # DEBUG
start_time = time.time()
TEXT_IDX = 1
SCORE_IDX = 2
CHUNK_IDX = 3
try:
ranker_answers = ranker([i['question'] for i in dataset])
save_json(ranker_answers, args.ranker_answers_path)
del ranker
def interact_model_by_telegram(model_config_path, token):
model_config = read_json(model_config_path)
model_config_key = model_config['metadata']['labels']['telegram_utils']
model = build_model_from_config(model_config)
init_bot_for_model(token, model, model_config_key)
def __init__(self, ranker_config=None, top_n=10, type='paragraph', active: bool = True,
batch_size=2, **kwargs):
"""
Args:
ranker: squad ranker model
top_n: number of documents to return
mode: paragraph or sentences
"""
self.ranker_config = json.load(expand_path(Path(ranker_config)).open())
self.ranker = build_model_from_config(self.ranker_config)
self.batch_size = batch_size
self.top_n = top_n
self.type = type
self.active = active
def main():
args = parser.parse_args()
odqa = build_model_from_config(read_json(args.config_path))
dataset = read_json(args.dataset_path)
dataset = dataset['data'][0]['paragraphs']
# dataset = dataset[:3]
# Select 20 random samples:
# dataset = random.sample(dataset, 50)
qa_dataset_size = len(dataset)
logger.info('QA dataset size: {}'.format(qa_dataset_size))
start_time = time.time()
try:
y_true_text = []
y_true_start = []
questions = []
def main():
args = parser.parse_args()
ranker_config = read_json(args.ranker_config_path)
reader_config = read_json(args.reader_config_path)
ranker = build_model_from_config(ranker_config) # chainer
reader = build_model_from_config(reader_config)
# print(reader([("Deep Pavlov killed Kenny.", "Who killed Kenny?")]))
dataset = read_json(args.dataset_path)
# dataset = dataset[:3]
qa_dataset_size = len(dataset)
logger.info('QA dataset size: {}'.format(qa_dataset_size))
# n_queries = 0 # DEBUG
start_time = time.time()
TEXT_IDX = 1
SQUAD_LIMIT = 500
SQUAD_BATCH_SIZE = 5
try:
mapping = {}
def main():
args = parser.parse_args()
config = read_json(args.config_path)
ranker = build_model_from_config(config) # chainer
dataset = read_json(args.dataset_path)
dataset_size = len(dataset)
logger.info('Dataset size: {}'.format(dataset_size))
n_correct_answers = 0
n_queries = 1
start_time = time.time()
try:
for instance in dataset:
q = instance['question']
q = unicodedata.normalize('NFD', q)
paragraphs = ranker([q])
answers = instance['answers']
formatted_answers = [encode_utf8(a) for a in answers]
def main():
args = parser.parse_args()
ranker_config = read_json(args.ranker_config_path)
reader_config = read_json(args.reader_config_path)
ranker = build_model_from_config(ranker_config) # chainer
reader = build_model_from_config(reader_config)
# print(reader([("Deep Pavlov killed Kenny.", "Who killed Kenny?")]))
dataset = read_json(args.dataset_path)
dataset = dataset[:3]
qa_dataset_size = len(dataset)
logger.info('QA dataset size: {}'.format(qa_dataset_size))
# n_queries = 0 # DEBUG
start_time = time.time()
TEXT_IDX = 1
# SQUAD_LIMIT = 500
SQUAD_BATCH_SIZE = 5
try:
mapping = {}