Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
SystemExit: If output folder exists and `force` is False.
# Returns
(str): The subfolder of the output path that contains the pypi package source.
"""
input_path = Path(input_dir)
output_path = Path(output_dir)
if meta_path is not None:
meta_path = Path(meta_path)
if not input_path or not input_path.exists():
msg.fail("Can't locate model data", input_path, exits=1)
if meta_path and not meta_path.exists():
msg.fail("Can't find model model.config.json", meta_path, exits=1)
meta_path = meta_path or input_path / "model.config.json"
if meta_path.is_file():
meta = srsly.read_json(meta_path)
msg.good("Loaded model.config.json from file", meta_path)
meta["mathy_version"] = f">={about.__version__},<1.0.0"
meta["name"] = model_name
for key in REQUIRED_META_KEYS:
if key not in meta or meta[key] == "":
msg.fail(
"No '{}' setting found in model.config.json".format(key),
"This setting is required to build your package.",
exits=1,
)
main_path = output_path / model_name
package_path = main_path
if package_path.exists():
if force:
shutil.rmtree(str(package_path))
else:
def get_model_meta(path):
"""Get model meta.json from a directory path and validate its contents.
path (unicode or Path): Path to model directory.
RETURNS (dict): The model's meta data.
"""
model_path = ensure_path(path)
if not model_path.exists():
raise IOError(Errors.E052.format(path=path2str(model_path)))
meta_path = model_path / "meta.json"
if not meta_path.is_file():
raise IOError(Errors.E053.format(path=meta_path))
meta = srsly.read_json(meta_path)
for setting in ["lang", "name", "version"]:
if setting not in meta or not meta[setting]:
raise ValueError(Errors.E054.format(setting=setting))
return meta
# Raises
ValueError: If **model_path** does not point to a valid folder.
ValueError: If **model_path** does not have a `model.config.json` file.
ValueError: If any required settings are missing from the meta file.
# Returns
(dict): The model's meta data.
"""
model_path = Path(model_path)
if not model_path.exists():
raise ValueError(f"cannot get meta from invalid model path: {model_path}")
meta_path = model_path / "model.config.json"
if not meta_path.is_file():
raise ValueError(f"invalid meta file: {meta_path}")
meta = srsly.read_json(meta_path)
for setting in REQUIRED_META_KEYS:
if setting not in meta or not meta[setting]:
raise ValueError(f"meta file missing required setting: {setting}")
return meta
# Raises
ValueError: If **model_path** does not point to a valid folder.
ValueError: If **model_path** does not have a `model.config.json` file.
ValueError: If any required settings are missing from the meta file.
# Returns
(dict): The model's meta data.
"""
model_path = Path(model_path)
if not model_path.exists():
raise ValueError(f"cannot get meta from invalid model path: {model_path}")
meta_path = model_path / "model.config.json"
if not meta_path.is_file():
raise ValueError(f"invalid meta file: {meta_path}")
meta = srsly.read_json(meta_path)
for setting in REQUIRED_META_KEYS:
if setting not in meta or not meta[setting]:
raise ValueError(f"meta file missing required setting: {setting}")
return meta
def gzip_language_data(root, source):
print("Compressing language data")
import srsly
from pathlib import Path
base = Path(root) / source
for jsonfile in base.glob("**/*.json"):
outfile = jsonfile.with_suffix(jsonfile.suffix + ".gz")
if outfile.is_file() and outfile.stat().st_mtime > jsonfile.stat().st_mtime:
# If the gz is newer it doesn't need updating
print("Skipping {}, already compressed".format(jsonfile))
continue
data = srsly.read_json(jsonfile)
srsly.write_gzip_json(outfile, data)
print("Compressed {}".format(jsonfile))
def from_disk(self, path: Union[Path, str], exclude: Sequence[str] = tuple()):
"""Load a Sense2Vec object from a directory.
path (unicode / Path): The path to load from.
exclude (list): Names of serialization fields to exclude.
RETURNS (Sense2Vec): The loaded object.
"""
path = Path(path)
strings_path = path / "strings.json"
index_path = path / "index.ann"
freqs_path = path / "freqs.json"
self.vectors = Vectors().from_disk(path)
self.cfg.update(srsly.read_json(path / "cfg"))
if freqs_path.exists():
self.freqs = dict(srsly.read_json(freqs_path))
if "strings" not in exclude and strings_path.exists():
self.strings = StringStore().from_disk(strings_path)
if "index" not in exclude and index_path.exists():
self.index = AnnoyIndex(self.vectors.shape[1], self.cfg["annoy_metric"])
self.index.load(str(index_path))
return self
deserializers["meta.json"] = lambda p: self.meta.update(srsly.read_json(p))
deserializers["vocab"] = lambda p: self.vocab.from_disk(
set and a meta.json already exists in the output directory, the existing
values will be used as the defaults in the command-line prompt.
"""
input_path = util.ensure_path(input_dir)
output_path = util.ensure_path(output_dir)
meta_path = util.ensure_path(meta_path)
if not input_path or not input_path.exists():
msg.fail("Can't locate model data", input_path, exits=1)
if not output_path or not output_path.exists():
msg.fail("Output directory not found", output_path, exits=1)
if meta_path and not meta_path.exists():
msg.fail("Can't find model meta.json", meta_path, exits=1)
meta_path = meta_path or input_path / "meta.json"
if meta_path.is_file():
meta = srsly.read_json(meta_path)
if not create_meta: # only print if user doesn't want to overwrite
msg.good("Loaded meta.json from file", meta_path)
else:
meta = generate_meta(input_dir, meta, msg)
for key in ("lang", "name", "version"):
if key not in meta or meta[key] == "":
msg.fail(
"No '{}' setting found in meta.json".format(key),
"This setting is required to build your package.",
exits=1,
)
model_name = meta["lang"] + "_" + meta["name"]
model_name_v = model_name + "-" + meta["version"]
main_path = output_path / model_name_v
package_path = main_path / model_name
def load_policy_value_model(
model_data_folder: str, silent: bool = False
) -> Tuple[PolicyValueModel, BaseConfig]:
meta_file = Path(model_data_folder) / "model.config.json"
if not meta_file.exists():
raise ValueError(f"model meta not found: {meta_file}")
args = BaseConfig(**srsly.read_json(str(meta_file)))
model_file = Path(model_data_folder) / "model.h5"
optimizer_file = Path(model_data_folder) / "model.optimizer"
if not model_file.exists():
raise ValueError(f"model not found: {model_file}")
if not optimizer_file.exists():
raise ValueError(f"optimizer not found: {optimizer_file}")
env: MathyEnv = PolySimplify()
observation: MathyObservation = env.state_to_observation(env.get_initial_state()[0])
initial_state: MathyWindowObservation = observations_to_window([observation])
model = PolicyValueModel(args=args, predictions=env.action_size, name="agent")
init_inputs = initial_state.to_inputs()
model.compile(
optimizer=model.optimizer, loss="binary_crossentropy", metrics=["accuracy"]
)
model.build(initial_state.to_input_shapes())
model.predict(init_inputs)