Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __init__(self,
map_filename: str = None,
last_turn_only: bool = True,
**kwargs):
map_filename = expand_path(map_filename)
self.map = pickle.load(open(map_filename, 'rb'))
self.last_turn_only = last_turn_only
def change_savepath_for_model(config):
params_helper = ParamsSearch()
dirs_for_saved_models = set()
for p in params_helper.find_model_path(config, SAVE_PATH_ELEMENT_NAME):
p.append(SAVE_PATH_ELEMENT_NAME)
save_path = Path(params_helper.get_value_from_config(config, p))
new_save_path = save_path.parent / TEMP_DIR_FOR_CV / save_path.name
dirs_for_saved_models.add(expand_path(new_save_path.parent))
params_helper.insert_value_or_dict_into_config(config, p, str(new_save_path))
return config, dirs_for_saved_models
def __init__(self, save_path: Optional[Union[str, Path]], load_path: Optional[Union[str, Path]] = None, mode: str = 'infer',
*args, **kwargs) -> None:
if save_path:
self.save_path = expand_path(save_path)
self.save_path.parent.mkdir(parents=True, exist_ok=True)
else:
self.save_path = None
if load_path:
self.load_path = expand_path(load_path)
if mode != 'train' and self.save_path and self.load_path != self.save_path:
log.warning("Load path '{}' differs from save path '{}' in '{}' mode for {}."
.format(self.load_path, self.save_path, mode, self.__class__.__name__))
elif mode != 'train' and self.save_path:
self.load_path = self.save_path
log.warning("No load path is set for {} in '{}' mode. Using save path instead"
.format(self.__class__.__name__, mode))
else:
self.load_path = None
log.warning("No load path is set for {}!".format(self.__class__.__name__))
def load(self) -> None:
"""Load classifier parameters"""
log.info("Loading from {}".format(self.load_path))
self.ec_data, self.x_train_features = load_pickle(
expand_path(self.load_path))
clip_norm=self.grad_clip)
self.sess.run(tf.global_variables_initializer())
self.summary_writer = None
self.step = 0
super().__init__(**kwargs)
self.run_id = str(uuid.uuid4())
self.log_path = str(expand_path(Path(self.opt['save_path']) / '{}'.format(self.run_id)))
if kwargs['mode'] == 'train':
if self.load_path is None:
self.save_path = expand_path(Path(self.opt['save_path']) / '{}_model'.format(self.run_id) / 'model')
else:
self.save_path = self.load_path
# Try to load the model (if there are some model files the model will be loaded from them)
if self.load_path is not None:
self.load()
def __init__(self, embedding_dim, max_sequence_length,
embeddings_path, save_path, load_path, embeddings="word2vec", seed=None):
"""Initialize the class according to given parameters."""
np.random.seed(seed)
save_path = expand_path(save_path).resolve().parent
load_path = expand_path(load_path).resolve().parent
self.int2emb_save_path = save_path / "int2emb.dict"
self.int2emb_load_path = load_path / "int2emb.dict"
self.embeddings = embeddings
self.embedding_dim = embedding_dim
self.max_sequence_length = max_sequence_length
self.emb_model_file = expand_path(embeddings_path)
self.int2emb_vocab = {}
def __init__(self, load_path, dict_path, top_k=10, **kwargs):
self.model = load_model(str(expand_path(load_path)),
custom_objects={"_margin_loss": self._margin_loss})
self.word_dict = self._build_dict(str(expand_path(dict_path)))
self.top_k = top_k
def __init__(self, emb_folder, emb_url, save_path, load_path,
x_len_limit, persona_len_limit, y_len_limit, char_limit, level='token', *args, **kwargs):
self.emb_folder = expand_path(emb_folder)
self.level = level
self.emb_url = emb_url
self.emb_file_name = Path(emb_url).name
self.save_path = expand_path(save_path)
self.load_path = expand_path(load_path)
self.x_len_limit = x_len_limit
self.persona_len_limit = persona_len_limit
self.y_len_limit = y_len_limit
self.char_limit = char_limit
self.loaded = False
self.NULL = ""
self.OOV = ""
self.emb_folder.mkdir(parents=True, exist_ok=True)
self.bert_config.attention_probs_dropout_prob = 1.0 - attention_probs_keep_prob
if hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - hidden_keep_prob
self.sess_config = tf.ConfigProto(allow_soft_placement=True)
self.sess_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=self.sess_config)
self._init_graph()
self._init_optimizer()
self.sess.run(tf.global_variables_initializer())
if pretrained_bert is not None:
pretrained_bert = str(expand_path(pretrained_bert))
if tf.train.checkpoint_exists(pretrained_bert) \
and not tf.train.checkpoint_exists(str(self.load_path.resolve())):
logger.info('[initializing model with Bert from {}]'.format(pretrained_bert))
var_list = self._get_saveable_variables(
exclude_scopes=('Optimizer', 'learning_rate', 'momentum', 'squad'))
saver = tf.train.Saver(var_list)
saver.restore(self.sess, pretrained_bert)
if self.load_path is not None:
self.load()