How to use the catalyst.utils.paths.ensure_directory function in catalyst

To help you get started, we’ve selected a few catalyst examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github enigmampc / catalyst / tests / exchange / test_bundle.py View on Github external
end_dt=end_dt
            )
        except Exception as e:
            log.warn('skipping ctable for {} from {} to {}: {}'.format(
                asset.symbol, start_dt, end_dt, e
            ))

        periods = bundle.get_calendar_periods_range(
            start_dt, end_dt, data_frequency
        )
        df = get_df_from_arrays(arrays, periods)

        folder = os.path.join(
            tempfile.gettempdir(), 'catalyst', exchange_name, asset.symbol
        )
        ensure_directory(folder)

        path = os.path.join(folder, filename + '.csv')

        log.info('creating csv file: {}'.format(path))
        print('HEAD\n{}'.format(df.head(100)))
        print('TAIL\n{}'.format(df.tail(100)))
        df.to_csv(path)
        pass
github enigmampc / catalyst / catalyst / utils / cache.py View on Github external
if serialization == 'msgpack':
            self.serialize = pd.DataFrame.to_msgpack
            self.deserialize = pd.read_msgpack
            self._protocol = None
        else:
            s = serialization.split(':', 1)
            if s[0] != 'pickle':
                raise ValueError(
                    "'serialization' must be either 'msgpack' or 'pickle[:n]'",
                )
            self._protocol = int(s[1]) if len(s) == 2 else None

            self.serialize = self._serialize_pickle
            self.deserialize = pickle.load

        ensure_directory(self.path)
github enigmampc / catalyst / catalyst / exchange / utils / exchange_utils.py View on Github external
Parameters
    ----------
    algo_name: str
    key: str
    obj: Object
    environ:
    rel_path: str
    how: str

    """
    folder = get_algo_folder(algo_name, environ)

    if rel_path is not None:
        folder = os.path.join(folder, rel_path)
        ensure_directory(folder)

    if how == 'json':
        filename = os.path.join(folder, '{}.json'.format(key))
        with open(filename, 'wt') as handle:
            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)

    else:
        filename = os.path.join(folder, '{}.p'.format(key))
        with open(filename, 'wb') as handle:
            pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
github enigmampc / catalyst / catalyst / exchange / utils / exchange_utils.py View on Github external
Parameters
    ----------
    algo_name: str
    environ:

    Returns
    -------
    str

    """
    if not environ:
        environ = os.environ

    root = data_root(environ)
    algo_folder = os.path.join(root, 'live_algos', algo_name)
    ensure_directory(algo_folder)

    return algo_folder
github enigmampc / catalyst / catalyst / exchange / utils / exchange_utils.py View on Github external
"""
    The temp folder for bundle downloads by algo name.

    Parameters
    ----------
    exchange_name: str

    Returns
    -------
    str

    """
    exchange_folder = get_exchange_folder(exchange_name)

    temp_bundles = os.path.join(exchange_folder, 'temp_bundles')
    ensure_directory(temp_bundles)

    return temp_bundles
github enigmampc / catalyst / catalyst / marketplace / utils / path_utils.py View on Github external
Parameters
    ----------
    environ:

    Returns
    -------
    str

    """
    if not environ:
        environ = os.environ

    root = data_root(environ)
    marketplace_folder = os.path.join(root, 'marketplace')
    ensure_directory(marketplace_folder)

    return marketplace_folder
github enigmampc / catalyst / catalyst / exchange / exchange_bundle.py View on Github external
Returns
        -------
        BcolzMinuteBarWriter | BcolzDailyBarWriter

        """
        root = get_exchange_folder(self.exchange_name)
        path = BUNDLE_NAME_TEMPLATE.format(
            root=root,
            frequency=data_frequency
        )

        if path in self._writers:
            return self._writers[path]

        ensure_directory(path)

        if len(os.listdir(path)) > 0:

            metadata = BcolzMinuteBarMetadata.read(path)

            write_metadata = False
            if start_dt < metadata.start_session:
                write_metadata = True
                start_session = start_dt
            else:
                start_session = metadata.start_session

            if end_dt > metadata.end_session:
                write_metadata = True

                end_session = end_dt
github enigmampc / catalyst / catalyst / exchange / utils / exchange_utils.py View on Github external
The minute writer folder for the exchange.

    Parameters
    ----------
    exchange_name: str
    environ:

    Returns
    -------
    BcolzExchangeBarWriter

    """
    exchange_folder = get_exchange_folder(exchange_name, environ)

    minute_data_folder = os.path.join(exchange_folder, 'minute_data')
    ensure_directory(minute_data_folder)

    return minute_data_folder
github enigmampc / catalyst / catalyst / exchange / utils / exchange_utils.py View on Github external
"""
    The root path of an exchange folder.

    Parameters
    ----------
    exchange_name: str

    Returns
    -------
    str

    """

    root = data_root()
    exchange_folder = os.path.join(root, 'exchanges', exchange_name)
    ensure_directory(exchange_folder)

    return exchange_folder
github enigmampc / catalyst / catalyst / exchange / utils / exchange_utils.py View on Github external
Parameters
    ----------
    algo_name: str
    key: str
    obj: Object
    environ:
    rel_path: str
    how: str

    """
    folder = get_algo_folder(algo_name, environ)

    if rel_path is not None:
        folder = os.path.join(folder, rel_path)
        ensure_directory(folder)

    if how == 'json':
        filename = os.path.join(folder, '{}.json'.format(key))
        with open(filename, 'wt') as handle:
            json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)

    else:
        filename = os.path.join(folder, '{}.p'.format(key))
        with open(filename, 'wb') as handle:
            pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)