How to use the mlrun.datastore.StoreManager function in mlrun

To help you get started, we’ve selected a few mlrun examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mlrun / mlrun / mlrun / run.py View on Github external
def download_object(url, target, secrets=None):
    """download mlrun dataitem (from path/url to target path)"""
    stores = StoreManager(secrets, db=get_run_db().connect())
    stores.object(url=url).download(target_path=target)
github mlrun / mlrun / mlrun / projects / project.py View on Github external
def _get_artifact_mngr(self):
        if self._artifact_mngr:
            return self._artifact_mngr
        db = get_run_db().connect(self._secrets)
        sm = StoreManager(self._secrets, db)
        self._artifact_mngr = ArtifactManager(sm, db)
        return self._artifact_mngr
github mlrun / mlrun / mlrun / db / filedb.py View on Github external
def connect(self, secrets=None):
        sm = StoreManager(secrets)
        self._datastore, self._subpath = sm.get_or_create_store(self.dirpath)
        return self
github mlrun / mlrun / mlrun / artifacts / model.py View on Github external
model_file, model_artifact, extra_data = get_model(models_path, suffix='.pkl')
        model = load(open(model_file, "rb"))
        categories = extra_data['categories'].as_df()

    :param model_dir:       model dir or artifact path (store://..) or DataItem
    :param suffix:          model filename suffix (when using a dir)
    :param stores:          StoreManager object (not required)

    :return model filename, model artifact object, extra data dict

    """
    model_file = ''
    model_spec = None
    extra_dataitems = {}
    suffix = suffix or '.pkl'
    stores = stores or StoreManager()
    if hasattr(model_dir, 'artifact_url'):
        model_dir = model_dir.artifact_url

    if model_dir.startswith(DB_SCHEMA + '://'):
        model_spec, target = stores.get_store_artifact(model_dir)
        if not model_spec or model_spec.kind != 'model':
            raise ValueError('store artifact ({}) is not model kind'.format(model_dir))
        model_file = _get_file_path(target, model_spec.model_file)
        extra_dataitems = _get_extra(stores, target, model_spec.extra_data)

    elif model_dir.lower().endswith('.yaml'):
        model_spec = _load_model_spec(model_dir, stores)
        model_file = _get_file_path(model_dir, model_spec.model_file)
        extra_dataitems = _get_extra(stores, model_dir, model_spec.extra_data)

    elif model_dir.endswith(suffix):
github mlrun / mlrun / mlrun / run.py View on Github external
def get_object(url, secrets=None, size=None, offset=0, db=None):
    """get mlrun dataitem body (from path/url)"""
    db = db or get_run_db().connect()
    stores = StoreManager(secrets, db=db)
    return stores.object(url=url).get(size, offset)
github mlrun / mlrun / mlrun / run.py View on Github external
def get_dataitem(url, secrets=None, db=None):
    """get mlrun dataitem object (from path/url)"""
    db = db or get_run_db().connect()
    stores = StoreManager(secrets, db=db)
    return stores.object(url=url)
github mlrun / mlrun / mlrun / artifacts / model.py View on Github external
:param model_artifact:  model artifact object or path (store://..) or DataItem
    :param parameters:      parameters dict
    :param metrics:         model metrics e.g. accuracy
    :param extra_data:      extra data items (key: path string | bytes | artifact)
    :param inputs:          list of inputs (feature vector schema)
    :param outputs:         list of outputs (output vector schema)
    :param key_prefix:      key prefix to add to metrics and extra data items
    :param labels:          metadata labels
    :param stores:          StoreManager object (not required)
    """

    if hasattr(model_artifact, 'artifact_url'):
        model_artifact = model_artifact.artifact_url

    stores = stores or StoreManager()
    if isinstance(model_artifact, ModelArtifact):
        model_spec = model_artifact
    elif model_artifact.startswith(DB_SCHEMA + '://'):
        model_spec, _ = stores.get_store_artifact(model_artifact)
    else:
        raise ValueError('model path must be a model store object/URL/DataItem')

    if not model_spec or model_spec.kind != 'model':
        raise ValueError('store artifact ({}) is not model kind'.format(model_artifact))

    if parameters:
        for key, val in parameters.items():
            model_spec.parameters[key] = val
    if metrics:
        for key, val in metrics.items():
            model_spec.metrics[key_prefix + key] = val