How to use the squad.squad_document_utils.random_filter_features function in squad

To help you get started, we’ve selected a few squad examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github huminghao16 / RE3QA / bert / run_triviaqa_wiki_full_e2e.py View on Github external
if os.path.exists(test_features_path):
        test_features = pickle.load(open(test_features_path, 'rb'))
        logger.info("Loading features from: %s" % (test_features_path))
    else:
        test_features = convert_examples_to_features(
            examples=test_examples,
            tokenizer=tokenizer,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            verbose_logging=args.verbose_logging,
            logger=logger)
        pickle.dump(test_features, open(test_features_path, 'wb'))

    logger.info("Filtering features randomly")
    filtered_test_features = random_filter_features(test_examples, test_features, args.n_best_size_rank,
                                                    is_training=False)
    filtered_rank_logits = [0.] * len(filtered_test_features)
    return build_eval_data(args, test_examples, test_features, filtered_test_features, filtered_rank_logits, logger)
github huminghao16 / RE3QA / bert / run_squad_document_full_e2e.py View on Github external
if os.path.exists(eval_features_path):
        eval_features = pickle.load(open(eval_features_path, 'rb'))
        logger.info("Loading features from: %s" % (eval_features_path))
    else:
        eval_features = convert_examples_to_features(
            examples=eval_examples,
            tokenizer=tokenizer,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            verbose_logging=args.verbose_logging,
            logger=logger)
        pickle.dump(eval_features, open(eval_features_path, 'wb'))

    logger.info("Filtering features randomly")
    filtered_eval_features = random_filter_features(eval_examples, eval_features, args.n_best_size_rank,
                                                    is_training=False)
    filtered_rank_logits = [0.] * len(filtered_eval_features)
    return build_eval_data(args, eval_examples, eval_features, filtered_eval_features, filtered_rank_logits, logger)
github huminghao16 / RE3QA / bert / run_squad_document_full_e2e.py View on Github external
train_features = convert_examples_to_features(
            examples=train_examples,
            tokenizer=tokenizer,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            verbose_logging=args.verbose_logging,
            logger=logger)
        pickle.dump(train_features, open(train_features_path, 'wb'))

    if args.down_sample:
        train_features = down_sample(args.sample_rate, train_features, logger)

    # filter features
    logger.info("Filtering features randomly")
    filtered_train_features = random_filter_features(train_examples, train_features, args.n_best_size_rank,
                                                     is_training=True)
    return build_train_data(args, train_examples, train_features, filtered_train_features, logger)
github huminghao16 / RE3QA / bert / run_triviaqa_wiki_full_e2e.py View on Github external
train_features = pickle.load(open(train_features_path, 'rb'))
        logger.info("Loading features from: %s" % (train_features_path))
    else:
        train_features = convert_examples_to_features(
            examples=train_examples,
            tokenizer=tokenizer,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            verbose_logging=args.verbose_logging,
            logger=logger)
        pickle.dump(train_features, open(train_features_path, 'wb'))

    # filter features
    logger.info("Filtering features randomly")
    filtered_train_features = random_filter_features(train_examples, train_features, args.n_best_size_rank,
                                                     is_training=True)
    return build_train_data(args, train_examples, train_features, filtered_train_features, logger)
github huminghao16 / RE3QA / bert / run_squad_document_full_e2e.py View on Github external
if os.path.exists(eval_features_path):
        eval_features = pickle.load(open(eval_features_path, 'rb'))
        logger.info("Loading features from: %s" % (eval_features_path))
    else:
        eval_features = convert_examples_to_features(
            examples=eval_examples,
            tokenizer=tokenizer,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            verbose_logging=args.verbose_logging,
            logger=logger)
        pickle.dump(eval_features, open(eval_features_path, 'wb'))

    logger.info("Filtering features randomly")
    filtered_eval_features = random_filter_features(eval_examples, eval_features, args.n_best_size_rank,
                                                    is_training=False)
    filtered_rank_logits = [0.] * len(filtered_eval_features)
    return build_eval_data(args, eval_examples, eval_features, filtered_eval_features, filtered_rank_logits, logger)
github huminghao16 / RE3QA / bert / run_triviaqa_wiki_full_e2e.py View on Github external
if os.path.exists(dev_features_path):
        dev_features = pickle.load(open(dev_features_path, 'rb'))
        logger.info("Loading features from: %s" % (dev_features_path))
    else:
        dev_features = convert_examples_to_features(
            examples=dev_examples,
            tokenizer=tokenizer,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            verbose_logging=args.verbose_logging,
            logger=logger)
        pickle.dump(dev_features, open(dev_features_path, 'wb'))

    logger.info("Filtering features randomly")
    filtered_dev_features = random_filter_features(dev_examples, dev_features, args.n_best_size_rank,
                                                   is_training=False)
    filtered_rank_logits = [0.] * len(filtered_dev_features)
    return build_eval_data(args, dev_examples, dev_features, filtered_dev_features, filtered_rank_logits, logger)