How to use the srsly.read_json function in srsly

To help you get started, we’ve selected a few srsly examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github justindujardin / mathy / libraries / mathy_python / mathy / models.py View on Github external
SystemExit: If output folder exists and `force` is False.

    # Returns
    (str): The subfolder of the output path that contains the pypi package source.
    """
    input_path = Path(input_dir)
    output_path = Path(output_dir)
    if meta_path is not None:
        meta_path = Path(meta_path)
    if not input_path or not input_path.exists():
        msg.fail("Can't locate model data", input_path, exits=1)
    if meta_path and not meta_path.exists():
        msg.fail("Can't find model model.config.json", meta_path, exits=1)
    meta_path = meta_path or input_path / "model.config.json"
    if meta_path.is_file():
        meta = srsly.read_json(meta_path)
        msg.good("Loaded model.config.json from file", meta_path)
    meta["mathy_version"] = f">={about.__version__},<1.0.0"
    meta["name"] = model_name
    for key in REQUIRED_META_KEYS:
        if key not in meta or meta[key] == "":
            msg.fail(
                "No '{}' setting found in model.config.json".format(key),
                "This setting is required to build your package.",
                exits=1,
            )
    main_path = output_path / model_name
    package_path = main_path
    if package_path.exists():
        if force:
            shutil.rmtree(str(package_path))
        else:
github explosion / spaCy / spacy / util.py View on Github external
def get_model_meta(path):
    """Get model meta.json from a directory path and validate its contents.

    path (unicode or Path): Path to model directory.
    RETURNS (dict): The model's meta data.
    """
    model_path = ensure_path(path)
    if not model_path.exists():
        raise IOError(Errors.E052.format(path=path2str(model_path)))
    meta_path = model_path / "meta.json"
    if not meta_path.is_file():
        raise IOError(Errors.E053.format(path=meta_path))
    meta = srsly.read_json(meta_path)
    for setting in ["lang", "name", "version"]:
        if setting not in meta or not meta[setting]:
            raise ValueError(Errors.E054.format(setting=setting))
    return meta
github justindujardin / mathy / libraries / mathy_python / mathy / models.py View on Github external
# Raises
    ValueError: If **model_path** does not point to a valid folder.
    ValueError: If **model_path** does not have a `model.config.json` file.
    ValueError: If any required settings are missing from the meta file.

    # Returns
    (dict): The model's meta data.
    """
    model_path = Path(model_path)
    if not model_path.exists():
        raise ValueError(f"cannot get meta from invalid model path: {model_path}")
    meta_path = model_path / "model.config.json"
    if not meta_path.is_file():
        raise ValueError(f"invalid meta file: {meta_path}")
    meta = srsly.read_json(meta_path)
    for setting in REQUIRED_META_KEYS:
        if setting not in meta or not meta[setting]:
            raise ValueError(f"meta file missing required setting: {setting}")
    return meta
github justindujardin / mathy / libraries / mathy_python / mathy / models.py View on Github external
# Raises
    ValueError: If **model_path** does not point to a valid folder.
    ValueError: If **model_path** does not have a `model.config.json` file.
    ValueError: If any required settings are missing from the meta file.

    # Returns
    (dict): The model's meta data.
    """
    model_path = Path(model_path)
    if not model_path.exists():
        raise ValueError(f"cannot get meta from invalid model path: {model_path}")
    meta_path = model_path / "model.config.json"
    if not meta_path.is_file():
        raise ValueError(f"invalid meta file: {meta_path}")
    meta = srsly.read_json(meta_path)
    for setting in REQUIRED_META_KEYS:
        if setting not in meta or not meta[setting]:
            raise ValueError(f"meta file missing required setting: {setting}")
    return meta
github explosion / spacy-lookups-data / setup.py View on Github external
def gzip_language_data(root, source):
    print("Compressing language data")
    import srsly
    from pathlib import Path

    base = Path(root) / source
    for jsonfile in base.glob("**/*.json"):
        outfile = jsonfile.with_suffix(jsonfile.suffix + ".gz")
        if outfile.is_file() and outfile.stat().st_mtime > jsonfile.stat().st_mtime:
            # If the gz is newer it doesn't need updating
            print("Skipping {}, already compressed".format(jsonfile))
            continue
        data = srsly.read_json(jsonfile)
        srsly.write_gzip_json(outfile, data)
        print("Compressed {}".format(jsonfile))
github explosion / sense2vec / sense2vec / sense2vec.py View on Github external
def from_disk(self, path: Union[Path, str], exclude: Sequence[str] = tuple()):
        """Load a Sense2Vec object from a directory.

        path (unicode / Path): The path to load from.
        exclude (list): Names of serialization fields to exclude.
        RETURNS (Sense2Vec): The loaded object.
        """
        path = Path(path)
        strings_path = path / "strings.json"
        index_path = path / "index.ann"
        freqs_path = path / "freqs.json"
        self.vectors = Vectors().from_disk(path)
        self.cfg.update(srsly.read_json(path / "cfg"))
        if freqs_path.exists():
            self.freqs = dict(srsly.read_json(freqs_path))
        if "strings" not in exclude and strings_path.exists():
            self.strings = StringStore().from_disk(strings_path)
        if "index" not in exclude and index_path.exists():
            self.index = AnnoyIndex(self.vectors.shape[1], self.cfg["annoy_metric"])
            self.index.load(str(index_path))
        return self
github explosion / spaCy / spacy / language.py View on Github external
        deserializers["meta.json"] = lambda p: self.meta.update(srsly.read_json(p))
        deserializers["vocab"] = lambda p: self.vocab.from_disk(
github explosion / spaCy / spacy / cli / package.py View on Github external
set and a meta.json already exists in the output directory, the existing
    values will be used as the defaults in the command-line prompt.
    """
    input_path = util.ensure_path(input_dir)
    output_path = util.ensure_path(output_dir)
    meta_path = util.ensure_path(meta_path)
    if not input_path or not input_path.exists():
        msg.fail("Can't locate model data", input_path, exits=1)
    if not output_path or not output_path.exists():
        msg.fail("Output directory not found", output_path, exits=1)
    if meta_path and not meta_path.exists():
        msg.fail("Can't find model meta.json", meta_path, exits=1)

    meta_path = meta_path or input_path / "meta.json"
    if meta_path.is_file():
        meta = srsly.read_json(meta_path)
        if not create_meta:  # only print if user doesn't want to overwrite
            msg.good("Loaded meta.json from file", meta_path)
        else:
            meta = generate_meta(input_dir, meta, msg)
    for key in ("lang", "name", "version"):
        if key not in meta or meta[key] == "":
            msg.fail(
                "No '{}' setting found in meta.json".format(key),
                "This setting is required to build your package.",
                exits=1,
            )
    model_name = meta["lang"] + "_" + meta["name"]
    model_name_v = model_name + "-" + meta["version"]
    main_path = output_path / model_name_v
    package_path = main_path / model_name
github justindujardin / mathy / libraries / mathy_python / mathy / agents / policy_value_model.py View on Github external
def load_policy_value_model(
    model_data_folder: str, silent: bool = False
) -> Tuple[PolicyValueModel, BaseConfig]:
    meta_file = Path(model_data_folder) / "model.config.json"
    if not meta_file.exists():
        raise ValueError(f"model meta not found: {meta_file}")
    args = BaseConfig(**srsly.read_json(str(meta_file)))
    model_file = Path(model_data_folder) / "model.h5"
    optimizer_file = Path(model_data_folder) / "model.optimizer"
    if not model_file.exists():
        raise ValueError(f"model not found: {model_file}")
    if not optimizer_file.exists():
        raise ValueError(f"optimizer not found: {optimizer_file}")
    env: MathyEnv = PolySimplify()
    observation: MathyObservation = env.state_to_observation(env.get_initial_state()[0])
    initial_state: MathyWindowObservation = observations_to_window([observation])
    model = PolicyValueModel(args=args, predictions=env.action_size, name="agent")
    init_inputs = initial_state.to_inputs()
    model.compile(
        optimizer=model.optimizer, loss="binary_crossentropy", metrics=["accuracy"]
    )
    model.build(initial_state.to_input_shapes())
    model.predict(init_inputs)