Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
storage_ok = True # default state
# if it does not exist try to create it
if not os.path.exists(CONFIG.MINDSDB_STORAGE_PATH):
try:
self.log.info('{folder} does not exist, creating it now'.format(folder=CONFIG.MINDSDB_STORAGE_PATH))
path = Path(CONFIG.MINDSDB_STORAGE_PATH)
path.mkdir(exist_ok=True, parents=True)
except:
self.log.info(traceback.format_exc())
storage_ok = False
self.log.error('MindsDB storage foldler: {folder} does not exist and could not be created'.format(
folder=CONFIG.MINDSDB_STORAGE_PATH))
# If storage path is not writable, raise an exception as this can no longer be
if not os.access(CONFIG.MINDSDB_STORAGE_PATH, os.W_OK) or storage_ok == False:
error_message = '''Cannot write into storage path, please either set the config variable mindsdb.config.set('MINDSDB_STORAGE_PATH',
def __init__(self, name, root_folder=CONFIG.MINDSDB_STORAGE_PATH, log_level=CONFIG.DEFAULT_LOG_LEVEL, log_server=CONFIG.MINDSDB_SERVER_URL):
"""
This controller defines the API to a MindsDB 'mind', a mind is an object that can learn and predict from data
:param name: the namespace you want to identify this mind instance with
:param root_folder: the folder where you want to store this mind or load from
:param log_level: the desired log level
:param log_server: the url for a server that can accept log streams
"""
# initialize variables
self.name = name
self.root_folder = root_folder
self.uuid = str(uuid.uuid1())
self.predict_worker = None
def __init__(self, name, root_folder=CONFIG.MINDSDB_STORAGE_PATH, log_level=CONFIG.DEFAULT_LOG_LEVEL):
"""
This controller defines the API to a MindsDB 'mind', a mind is an object that can learn and predict from data
:param name: the namespace you want to identify this mind instance with
:param root_folder: the folder where you want to store this mind or load from
:param log_level: the desired log level
"""
# initialize variables
self.name = name
self.root_folder = root_folder
self.uuid = str(uuid.uuid1())
# initialize log
self.log = MindsdbLogger(log_level=log_level, uuid=self.uuid)
def check_for_updates():
"""
Check for updates of mindsdb
it will ask the mindsdb server if there are new versions, if there are it will log a message
:return: None
"""
# tmp files
uuid_file = CONFIG.MINDSDB_STORAGE_PATH + '/../uuid.mdb_base'
mdb_file = CONFIG.MINDSDB_STORAGE_PATH + '/start.mdb_base'
uuid_file_path = Path(uuid_file)
if uuid_file_path.is_file():
uuid_str = open(uuid_file).read()
else:
uuid_str = str(uuid.uuid4())
try:
open(uuid_file, 'w').write(uuid_str)
except:
log.warning('Cannot store token, Please add write permissions to file:' + uuid_file)
uuid_str = uuid_str + '.NO_WRITE'
file_path = Path(mdb_file)
if file_path.is_file():
token = open(mdb_file).read()
else:
def save_metadata(self):
fn = os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.lmd['name'] + '_light_model_metadata.pickle')
self.lmd['updated_at'] = str(datetime.datetime.now())
try:
with open(fn, 'wb') as fp:
pickle.dump(self.lmd, fp,protocol=pickle.HIGHEST_PROTOCOL)
except:
self.log.error(f'Could not save mindsdb heavy metadata in the file: {fn}')
fn = os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.hmd['name'] + '_heavy_model_metadata.pickle')
save_hmd = {}
null_out_fields = ['test_from_data', 'from_data']
for k in null_out_fields:
save_hmd[k] = None
for k in self.hmd:
if k not in null_out_fields:
if self.lmd['model_backend'] == 'ludwig':
self.lmd['is_active'] = True
self.model_backend = LudwigBackend(self)
self.model_backend.train()
self.lmd['is_active'] = False
self.lmd['train_end_at'] = str(datetime.datetime.now())
self._call_phase_module('ModelAnalyzer')
with open(CONFIG.MINDSDB_STORAGE_PATH + '/' + self.lmd['name'] + '_light_model_metadata.pickle', 'wb') as fp:
self.lmd['updated_at'] = str(datetime.datetime.now())
pickle.dump(self.lmd, fp)
with open(CONFIG.MINDSDB_STORAGE_PATH + '/' + self.hmd['name'] + '_heavy_model_metadata.pickle', 'wb') as fp:
# Don't save data for now
self.hmd['from_data'] = None
self.hmd['test_from_data'] = None
# Don't save data for now
pickle.dump(self.hmd, fp)
return
except Exception as e:
self.lmd['is_active'] = False
self.lmd['current_phase'] = MODEL_STATUS_ERROR
self.lmd['error_msg'] = traceback.print_exc()
self.log.error(str(e))
raise e
try:
lmd['ludwig_data']['ludwig_save_path'] = lmd['ludwig_data']['ludwig_save_path'].replace(old_model_name, new_model_name)
renamed_one_backend = True
except:
pass
try:
lmd['lightwood_data']['save_path'] = lmd['lightwood_data']['save_path'].replace(old_model_name, new_model_name)
renamed_one_backend = True
except:
pass
if not renamed_one_backend:
return False
with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, new_model_name + '_light_model_metadata.pickle'), 'wb') as fp:
pickle.dump(lmd, fp,protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, new_model_name + '_heavy_model_metadata.pickle'), 'wb') as fp:
pickle.dump(hmd, fp,protocol=pickle.HIGHEST_PROTOCOL)
os.remove(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, old_model_name + '_light_model_metadata.pickle'))
os.remove(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, old_model_name + '_heavy_model_metadata.pickle'))
return True
def get_model_data(self, model_name, lmd=None):
if lmd is None:
with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, f'{model_name}_light_model_metadata.pickle'), 'rb') as fp:
lmd = pickle.load(fp)
# ADAPTOR CODE
amd = {}
if lmd['current_phase'] == MODEL_STATUS_TRAINED:
amd['status'] = 'complete'
elif lmd['current_phase'] == MODEL_STATUS_ERROR:
amd['status'] = 'error'
else:
amd['status'] = 'training'
# Shared keys
for k in ['name', 'version', 'is_active', 'data_source', 'predict', 'current_phase',
'train_end_at', 'updated_at', 'created_at','data_preparation', 'validation_set_accuracy']:
if k == 'predict':
amd[k] = lmd['predict_columns']
def check_for_updates():
"""
Check for updates of mindsdb
it will ask the mindsdb server if there are new versions, if there are it will log a message
:return: None
"""
# tmp files
uuid_file = CONFIG.MINDSDB_STORAGE_PATH + '/../uuid.mdb_base'
mdb_file = CONFIG.MINDSDB_STORAGE_PATH + '/start.mdb_base'
uuid_file_path = Path(uuid_file)
if uuid_file_path.is_file():
uuid_str = open(uuid_file).read()
else:
uuid_str = str(uuid.uuid4())
try:
open(uuid_file, 'w').write(uuid_str)
except:
log.warning('Cannot store token, Please add write permissions to file:' + uuid_file)
uuid_str = uuid_str + '.NO_WRITE'
file_path = Path(mdb_file)
if file_path.is_file():
token = open(mdb_file).read()