How to use the deeppavlov.core.commands.utils.expand_path function in deeppavlov

To help you get started, we’ve selected a few deeppavlov examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github deepmipt / DeepPavlov / deeppavlov / models / ranking / search_index.py View on Github external
def __init__(self,
                 map_filename: str = None,
                 last_turn_only: bool = True,
                 **kwargs):
        map_filename = expand_path(map_filename)
        self.map = pickle.load(open(map_filename, 'rb'))
        self.last_turn_only = last_turn_only
github deepmipt / DeepPavlov / deeppavlov / core / common / cross_validation.py View on Github external
def change_savepath_for_model(config):
    params_helper = ParamsSearch()

    dirs_for_saved_models = set()
    for p in params_helper.find_model_path(config, SAVE_PATH_ELEMENT_NAME):
        p.append(SAVE_PATH_ELEMENT_NAME)
        save_path = Path(params_helper.get_value_from_config(config, p))
        new_save_path = save_path.parent / TEMP_DIR_FOR_CV / save_path.name

        dirs_for_saved_models.add(expand_path(new_save_path.parent))

        params_helper.insert_value_or_dict_into_config(config, p, str(new_save_path))

    return config, dirs_for_saved_models
github deepmipt / DeepPavlov / deeppavlov / core / models / serializable.py View on Github external
def __init__(self, save_path: Optional[Union[str, Path]], load_path: Optional[Union[str, Path]] = None, mode: str = 'infer',
                 *args, **kwargs) -> None:

        if save_path:
            self.save_path = expand_path(save_path)
            self.save_path.parent.mkdir(parents=True, exist_ok=True)
        else:
            self.save_path = None

        if load_path:
            self.load_path = expand_path(load_path)
            if mode != 'train' and self.save_path and self.load_path != self.save_path:
                log.warning("Load path '{}' differs from save path '{}' in '{}' mode for {}."
                            .format(self.load_path, self.save_path, mode, self.__class__.__name__))
        elif mode != 'train' and self.save_path:
            self.load_path = self.save_path
            log.warning("No load path is set for {} in '{}' mode. Using save path instead"
                        .format(self.__class__.__name__, mode))
        else:
            self.load_path = None
            log.warning("No load path is set for {}!".format(self.__class__.__name__))
github deepmipt / DeepPavlov / deeppavlov / skills / ecommerce_skill / tfidf_retrieve.py View on Github external
def load(self) -> None:
        """Load classifier parameters"""
        log.info("Loading from {}".format(self.load_path))
        self.ec_data, self.x_train_features = load_pickle(
            expand_path(self.load_path))
github deepmipt / DeepPavlov / deeppavlov / models / personachat / personachat_model.py View on Github external
clip_norm=self.grad_clip)

        self.sess.run(tf.global_variables_initializer())

        self.summary_writer = None

        self.step = 0

        super().__init__(**kwargs)

        self.run_id = str(uuid.uuid4())
        self.log_path = str(expand_path(Path(self.opt['save_path']) / '{}'.format(self.run_id)))

        if kwargs['mode'] == 'train':
            if self.load_path is None:
                self.save_path = expand_path(Path(self.opt['save_path']) / '{}_model'.format(self.run_id) / 'model')
            else:
                self.save_path = self.load_path
        # Try to load the model (if there are some model files the model will be loaded from them)
        if self.load_path is not None:
            self.load()
github deepmipt / DeepPavlov / deeppavlov / models / ranking / emb_dict.py View on Github external
def __init__(self, embedding_dim, max_sequence_length,
                 embeddings_path, save_path, load_path, embeddings="word2vec", seed=None):
        """Initialize the class according to given parameters."""
        np.random.seed(seed)
        save_path = expand_path(save_path).resolve().parent
        load_path = expand_path(load_path).resolve().parent
        self.int2emb_save_path = save_path / "int2emb.dict"
        self.int2emb_load_path = load_path / "int2emb.dict"
        self.embeddings = embeddings
        self.embedding_dim = embedding_dim
        self.max_sequence_length = max_sequence_length
        self.emb_model_file = expand_path(embeddings_path)

        self.int2emb_vocab = {}
github deepmipt / DeepPavlov / deeppavlov / skills / odqa / paragraph_ranker.py View on Github external
def __init__(self, load_path, dict_path, top_k=10, **kwargs):
        self.model = load_model(str(expand_path(load_path)),
                                custom_objects={"_margin_loss": self._margin_loss})
        self.word_dict = self._build_dict(str(expand_path(dict_path)))
        self.top_k = top_k
github deepmipt / DeepPavlov / deeppavlov / models / preprocessors / personachat_preprocessor.py View on Github external
def __init__(self, emb_folder, emb_url, save_path, load_path,
                 x_len_limit, persona_len_limit, y_len_limit, char_limit, level='token', *args, **kwargs):
        self.emb_folder = expand_path(emb_folder)
        self.level = level
        self.emb_url = emb_url
        self.emb_file_name = Path(emb_url).name
        self.save_path = expand_path(save_path)
        self.load_path = expand_path(load_path)
        self.x_len_limit = x_len_limit
        self.persona_len_limit = persona_len_limit
        self.y_len_limit = y_len_limit
        self.char_limit = char_limit
        self.loaded = False

        self.NULL = ""
        self.OOV = ""

        self.emb_folder.mkdir(parents=True, exist_ok=True)
github deepmipt / DeepPavlov / deeppavlov / models / bert / bert_squad.py View on Github external
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
        if hidden_keep_prob is not None:
            self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob

        self.sess_config = tf.ConfigProto(allow_soft_placement=True)
        self.sess_config.gpu_options.allow_growth = True
        self.sess = tf.Session(config=self.sess_config)

        self._init_graph()

        self._init_optimizer()

        self.sess.run(tf.global_variables_initializer())

        if pretrained_bert is not None:
            pretrained_bert = str(expand_path(pretrained_bert))

            if tf.train.checkpoint_exists(pretrained_bert) \
                    and not tf.train.checkpoint_exists(str(self.load_path.resolve())):
                logger.info('[initializing model with Bert from {}]'.format(pretrained_bert))
                var_list = self._get_saveable_variables(
                    exclude_scopes=('Optimizer', 'learning_rate', 'momentum', 'squad'))
                saver = tf.train.Saver(var_list)
                saver.restore(self.sess, pretrained_bert)

        if self.load_path is not None:
            self.load()