How to use the deeppavlov.core.commands.infer.build_model_from_config function in deeppavlov

To help you get started, we’ve selected a few deeppavlov examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github deepmipt / DeepPavlov / deeppavlov / core / commands / train.py View on Github external
_fit_batches(model, iterator, train_config)
        elif callable(getattr(model, 'fit', None)):
            _fit(model, iterator, train_config)
        elif not isinstance(model, Chainer):
            log.warning('Nothing to train')

        model.destroy()

    res = {}

    if train_config['validate_best'] or train_config['test_best']:
        # try:
        #     model_config['load_path'] = model_config['save_path']
        # except KeyError:
        #     log.warning('No "save_path" parameter for the model, so "load_path" will not be renewed')
        model = build_model_from_config(config, load_trained=True)
        log.info('Testing the best saved model')

        if train_config['validate_best']:
            report = {
                'valid': _test_model(model, metrics_functions, iterator,
                                     train_config.get('batch_size', -1), 'valid',
                                     show_examples=train_config['show_examples'])
            }

            res['valid'] = report['valid']['metrics']

            print(json.dumps(report, ensure_ascii=False))

        if train_config['test_best']:
            report = {
                'test': _test_model(model, metrics_functions, iterator,
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / help_scripts / get_ranker12_output.py View on Github external
def main():
    args = parser.parse_args()
    config = read_json(args.config_path)
    ranker = build_model_from_config(config)  # chainer
    dataset = read_csv(args.dataset_path)
    output_path = args.output_path

    with open(output_path, 'w') as fout:
        result = []
        i = 0
        for instance in dataset:
            print('Processing {} query'.format(i))
            q = instance['question']
            a = instance['answer']
            pred = ranker([q])
            result.append({'question': q, 'answer': a, 'context': pred})
            i += 1
        json.dump(result, fout)
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / help_scripts / wiki_popularity / gather_popularity_dataset.py View on Github external
def main():
    args = parser.parse_args()
    config = read_json(args.config_path)
    ranker = build_model_from_config(config)  # chainer
    dataset = read_json(args.dataset_path)
    # dataset = dataset[:10]

    qa_dataset_size = len(dataset)
    logger.info('QA dataset size: {}'.format(qa_dataset_size))
    # n_queries = 0  # DEBUG
    start_time = time.time()
    TEXT_IDX = 1
    SCORE_IDX = 2
    CHUNK_IDX = 3

    try:

        ranker_answers = ranker([i['question'] for i in dataset])
        save_json(ranker_answers, args.ranker_answers_path)
        del ranker
github deepmipt / DeepPavlov / utils / telegram_utils / telegram_ui.py View on Github external
def interact_model_by_telegram(model_config_path, token):
    model_config = read_json(model_config_path)
    model_config_key = model_config['metadata']['labels']['telegram_utils']
    model = build_model_from_config(model_config)
    init_bot_for_model(token, model, model_config_key)
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / squad_paragraph_ranker.py View on Github external
def __init__(self, ranker_config=None, top_n=10, type='paragraph', active: bool = True,
                 batch_size=2, **kwargs):
        """

        Args:
            ranker: squad ranker model
            top_n: number of documents to return
            mode: paragraph or sentences
        """
        self.ranker_config = json.load(expand_path(Path(ranker_config)).open())
        self.ranker = build_model_from_config(self.ranker_config)
        self.batch_size = batch_size
        self.top_n = top_n
        self.type = type
        self.active = active
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / help_scripts / kpi_2018 / eval_kpi_10.py View on Github external
def main():
    args = parser.parse_args()
    odqa = build_model_from_config(read_json(args.config_path))
    dataset = read_json(args.dataset_path)
    dataset = dataset['data'][0]['paragraphs']
    # dataset = dataset[:3]

    # Select 20 random samples:
    # dataset = random.sample(dataset, 50)

    qa_dataset_size = len(dataset)
    logger.info('QA dataset size: {}'.format(qa_dataset_size))
    start_time = time.time()

    try:

        y_true_text = []
        y_true_start = []
        questions = []
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / eval_scripts / evaluate_odqa_via_logits.py View on Github external
def main():
    args = parser.parse_args()
    ranker_config = read_json(args.ranker_config_path)
    reader_config = read_json(args.reader_config_path)
    ranker = build_model_from_config(ranker_config)  # chainer
    reader = build_model_from_config(reader_config)
    # print(reader([("Deep Pavlov killed Kenny.", "Who killed Kenny?")]))
    dataset = read_json(args.dataset_path)
    # dataset = dataset[:3]

    qa_dataset_size = len(dataset)
    logger.info('QA dataset size: {}'.format(qa_dataset_size))
    # n_queries = 0  # DEBUG
    start_time = time.time()
    TEXT_IDX = 1
    SQUAD_LIMIT = 500
    SQUAD_BATCH_SIZE = 5

    try:
        mapping = {}
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / eval_scripts / evaluate_paragraph_ranker_recall.py View on Github external
def main():
    args = parser.parse_args()
    config = read_json(args.config_path)
    ranker = build_model_from_config(config)  # chainer
    dataset = read_json(args.dataset_path)

    dataset_size = len(dataset)
    logger.info('Dataset size: {}'.format(dataset_size))
    n_correct_answers = 0
    n_queries = 1
    start_time = time.time()

    try:
        for instance in dataset:
            q = instance['question']
            q = unicodedata.normalize('NFD', q)
            paragraphs = ranker([q])

            answers = instance['answers']
            formatted_answers = [encode_utf8(a) for a in answers]
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / eval_scripts / evaluate_odqa_with_ensemble_ranker.py View on Github external
def main():
    args = parser.parse_args()
    ranker_config = read_json(args.ranker_config_path)
    reader_config = read_json(args.reader_config_path)
    ranker = build_model_from_config(ranker_config)  # chainer
    reader = build_model_from_config(reader_config)
    # print(reader([("Deep Pavlov killed Kenny.", "Who killed Kenny?")]))
    dataset = read_json(args.dataset_path)
    dataset = dataset[:3]

    qa_dataset_size = len(dataset)
    logger.info('QA dataset size: {}'.format(qa_dataset_size))
    # n_queries = 0  # DEBUG
    start_time = time.time()
    TEXT_IDX = 1
    # SQUAD_LIMIT = 500
    SQUAD_BATCH_SIZE = 5

    try:
        mapping = {}