Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
TRAIN_SET_METADATA_FILE_NAME
)
# preprocessing
dataset, train_set_metadata = preprocess_for_prediction(
model_path,
split,
data_csv,
data_hdf5,
train_set_metadata_fp
)
model, model_definition = load_model_and_definition(model_path)
# collect activations
print_boxed('COLLECT ACTIVATIONS')
collected_tensors = model.collect_activations(
dataset,
tensors,
batch_size,
gpus=gpus,
gpu_fraction=gpu_fraction
)
model.close_session()
# saving
os.makedirs(experiment_dir_name)
save_tensors(collected_tensors, experiment_dir_name)
logger.info('Saved to: {0}'.format(experiment_dir_name))
:param gpus: List of GPUs that are available for training.
:type gpus: List
:param gpu_fraction: Fraction of the memory of each GPU to use at
the beginning of the training. The memory may grow elastically.
:type gpu_fraction: Integer
:param random_seed: Random seed used for weights initialization,
splits and any other random function.
:type random_seed: Integer
:param debug: If true turns on tfdbg with inf_or_nan checks.
:type debug: Boolean
:returns: None
"""
if model_load_path is not None:
# Load model
if is_on_master():
print_boxed('LOADING MODEL')
logger.info('Loading model: {}\n'.format(model_load_path))
model, _ = load_model_and_definition(model_load_path)
else:
# Build model
if is_on_master():
print_boxed('BUILDING MODEL', print_fun=logger.debug)
model = Model(
model_definition['input_features'],
model_definition['output_features'],
model_definition['combiner'],
model_definition['training'],
model_definition['preprocessing'],
use_horovod=use_horovod,
random_seed=random_seed,
debug=debug
tensors,
output_directory='results',
debug=False,
**kwargs
):
# setup directories and file names
experiment_dir_name = find_non_existing_dir_by_adding_suffix(output_directory)
logger.info('Model path: {}'.format(model_path))
logger.info('Output path: {}'.format(experiment_dir_name))
logger.info('\n')
model, model_definition = load_model_and_definition(model_path)
# collect weights
print_boxed('COLLECT WEIGHTS')
collected_tensors = model.collect_weights(tensors)
model.close_session()
# saving
os.makedirs(experiment_dir_name)
save_tensors(collected_tensors, experiment_dir_name)
logger.info('Saved to: {0}'.format(experiment_dir_name))
model = Model(
model_definition['input_features'],
model_definition['output_features'],
model_definition['combiner'],
model_definition['training'],
model_definition['preprocessing'],
use_horovod=use_horovod,
random_seed=random_seed,
debug=debug
)
contrib_command("train_model", model, model_definition, model_load_path)
# Train model
if is_on_master():
print_boxed('TRAINING')
return model, model.train(
training_set,
validation_set=validation_set,
test_set=test_set,
save_path=save_path,
resume=resume,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
gpus=gpus, gpu_fraction=gpu_fraction,
random_seed=random_seed,
**model_definition['training']
)
splits and any other random function.
:type random_seed: Integer
:param debug: If true turns on tfdbg with inf_or_nan checks.
:type debug: Boolean
:returns: None
"""
if model_load_path is not None:
# Load model
if is_on_master():
print_boxed('LOADING MODEL')
logger.info('Loading model: {}\n'.format(model_load_path))
model, _ = load_model_and_definition(model_load_path)
else:
# Build model
if is_on_master():
print_boxed('BUILDING MODEL', print_fun=logger.debug)
model = Model(
model_definition['input_features'],
model_definition['output_features'],
model_definition['combiner'],
model_definition['training'],
model_definition['preprocessing'],
use_horovod=use_horovod,
random_seed=random_seed,
debug=debug
)
contrib_command("train_model", model, model_definition, model_load_path)
# Train model
if is_on_master():
TRAIN_SET_METADATA_FILE_NAME
)
# preprocessing
dataset, train_set_metadata = preprocess_for_prediction(
model_path,
split,
data_csv,
data_hdf5,
train_set_metadata_json_fp,
evaluate_performance
)
# run the prediction
if is_on_master():
print_boxed('LOADING MODEL')
model, model_definition = load_model_and_definition(model_path,
use_horovod=use_horovod)
prediction_results = predict(
dataset,
train_set_metadata,
model,
model_definition,
batch_size,
evaluate_performance,
gpus,
gpu_fraction,
debug
)
model.close_session()
will be returned, if it is True, also performance metrics
will be calculated on the predictions. It requires the data
to contain also ground truth for the output features, otherwise
the metrics cannot be computed.
:type evaluate_performance: Bool
:type gpus: List
:type gpu_fraction: Integer
:param debug: If true turns on tfdbg with inf_or_nan checks.
:type debug: Boolean
:returns: A dictionary containing the predictions of each output feature,
alongside with statistics on the quality of those predictions
(if evaluate_performance is True).
"""
if is_on_master():
print_boxed('PREDICT')
test_stats = model.predict(
dataset,
batch_size,
evaluate_performance=evaluate_performance,
gpus=gpus,
gpu_fraction=gpu_fraction
)
if evaluate_performance:
calculate_overall_stats(
test_stats,
model_definition['output_features'],
dataset,
train_set_metadata
)