Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def the_batch_centroid_is_finished_in_less_than(step, secs):
wait_until_batch_centroid_status_code_is(step, FINISHED, FAULTY, secs)
def the_sample_is_finished_in_less_than(step, secs):
wait_until_sample_status_code_is(step, FINISHED, FAULTY, secs)
def the_evaluation_is_finished_in_less_than(step, secs):
wait_until_evaluation_status_code_is(step, FINISHED, FAULTY, secs)
def the_cluster_is_finished_in_less_than(step, secs):
wait_until_cluster_status_code_is(step, FINISHED, FAULTY, secs)
def the_source_is_finished(step, secs):
wait_until_source_status_code_is(step, FINISHED, FAULTY, secs)
def retrieve_models_split(models_split, api, query_string=FIELDS_QS,
labels=None, multi_label_data=None, ordered=True,
models_order=None):
"""Returns a list of full model structures ready to be fed to the
MultiModel object to produce predictions. Models are also stored
locally in the output directory when the --store flag is used.
"""
complete_models = []
if models_order is None:
models_order = []
for model in models_split:
if (isinstance(model, basestring) or
bigml.api.get_status(model)['code'] != bigml.api.FINISHED):
try:
model = u.check_resource(model, api.get_model,
query_string)
except ValueError, exception:
sys.exit("Failed to get model: %s. %s" % (model,
str(exception)))
# When user selects the labels in multi-label predictions, we must
# filter the models that will be used to predict
if labels and multi_label_data:
objective_column = str(multi_label_data['objective_column'])
labels_info = multi_label_data[
'generated_fields'][objective_column]
labels_columns = [label_info[1] for label_info in labels_info
if label_info[0] in labels]
model_objective_id = model['object']['objective_fields'][0]
def get_source(source, api=None, verbosity=True,
session_file=None):
"""Retrieves the source in its actual state and its field info
"""
if api is None:
api = bigml.api.BigML()
if (isinstance(source, basestring) or
bigml.api.get_status(source)['code'] != bigml.api.FINISHED):
message = dated("Retrieving source. %s\n" %
get_url(source))
log_message(message, log_file=session_file,
console=verbosity)
try:
source = check_resource(source, api.get_source,
query_string=ALL_FIELDS_QS,
raise_on_error=True)
except Exception, exception:
sys.exit("Failed to get a finished source: %s" % str(exception))
return source
LOG_FILES = [COMMAND_LOG, DIRS_LOG, u.NEW_DIRS_LOG]
ROWS_LIMIT = 15
INDENT_IDS = 26
RESOURCES_LOG_FILES = set(['project', 'source', 'dataset', 'dataset_train',
'dataset_test', 'dataset_gen', 'dataset_cluster',
'dataset_parts', 'dataset_multi', 'models',
'ensembles', 'evaluations',
'clusters', 'batch_prediction', 'batch_centroid',
'anomalies', 'batch_anomaly_score', 'sample',
'associations', 'time_series', "deepnets",
'fusions', 'pcas', 'batch_projection',
'linear_regressions', 'logistic_regressions',
'scripts', 'library', 'execution',
'external_connector'])
STATUS_CODES = {
"finished": bigml.api.FINISHED,
"faulty": bigml.api.FAULTY,
"waiting": bigml.api.WAITING,
"queued": bigml.api.QUEUED,
"started": bigml.api.STARTED,
"in progress": bigml.api.IN_PROGRESS,
"summarized": bigml.api.SUMMARIZED,
"uploading": bigml.api.UPLOADING,
"unknown": bigml.api.UNKNOWN,
"runnable": bigml.api.RUNNABLE
}
GROUP_RESOURCES = ["project", "execution"]
COMPOSED_RESOURCES = ["cluster", "ensemble", "fusion", "composites"]
def retrieve_resources(directory):
self.resource_id = None
self.ids_map = {}
self.terms = {}
self.regression = False
self.boosting = None
self.class_names = None
self.api = get_api_connection(api)
self.resource_id, model = get_resource_dict( \
model, "model", api=self.api)
if 'object' in model and isinstance(model['object'], dict):
model = model['object']
if 'model' in model and isinstance(model['model'], dict):
status = get_status(model)
if 'code' in status and status['code'] == FINISHED:
self.input_fields = model["input_fields"]
BaseModel.__init__(self, model, api=api, fields=fields)
# boosting models are to be handled using the BoostedTree
# class
if model.get("boosted_ensemble"):
self.boosting = model.get('boosting', False)
if self.boosting == {}:
self.boosting = False
self.regression = \
not self.boosting and \
self.fields[self.objective_id]['optype'] == 'numeric' \
or (self.boosting and \
self.boosting.get("objective_class") is None)
anomaly, "anomaly", api=self.api)
if 'object' in anomaly and isinstance(anomaly['object'], dict):
anomaly = anomaly['object']
self.sample_size = anomaly.get('sample_size')
self.input_fields = anomaly.get('input_fields')
self.id_fields = anomaly.get('id_fields', [])
if 'model' in anomaly and isinstance(anomaly['model'], dict):
ModelFields.__init__( \
self, anomaly['model'].get('fields'), \
missing_tokens=anomaly['model'].get('missing_tokens'))
if ('top_anomalies' in anomaly['model'] and
isinstance(anomaly['model']['top_anomalies'], list)):
self.mean_depth = anomaly['model'].get('mean_depth')
status = get_status(anomaly)
if 'code' in status and status['code'] == FINISHED:
self.expected_mean_depth = None
if self.mean_depth is None or self.sample_size is None:
raise Exception("The anomaly data is not complete. "
"Score will"
" not be available")
else:
default_depth = (
2 * (DEPTH_FACTOR + \
math.log(self.sample_size - 1) - \
(float(self.sample_size - 1) / self.sample_size)))
self.expected_mean_depth = min(self.mean_depth,
default_depth)
iforest = anomaly['model'].get('trees', [])
if iforest:
self.iforest = [
AnomalyTree(anomaly_tree['root'], self.fields)