Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_good_set_before_init_no_hit(self, random_dt, database, new_config):
"""Trying to set, overwrite everything from input."""
exp = Experiment(new_config['name'])
exp.configure(new_config)
assert exp._init_done is True
found_config = list(database.experiments.find({'name': 'supernaekei',
'metadata.user': 'tsirif'}))
new_config['metadata']['datetime'] = exp.metadata['datetime']
assert len(found_config) == 1
_id = found_config[0].pop('_id')
assert _id != 'fasdfasfa'
assert exp._id == _id
new_config['refers'] = {}
new_config.pop('_id')
new_config.pop('something_to_be_ignored')
new_config['algorithms']['dumbalgo']['done'] = False
new_config['algorithms']['dumbalgo']['judgement'] = None
def test_new_experiment_with_parent(self, create_db_instance, random_dt, exp_config):
"""Configure a branch experiment."""
exp = Experiment('supernaedo2.6')
exp.metadata = exp_config[0][4]['metadata']
exp.refers = exp_config[0][4]['refers']
exp.algorithms = exp_config[0][4]['algorithms']
exp.configure(exp.configuration)
assert exp._init_done is True
assert_protocol(exp, create_db_instance)
assert exp._id is not None
assert exp.name == 'supernaedo2.6'
assert exp.configuration['refers'] == exp_config[0][4]['refers']
exp_config[0][4]['metadata']['datetime'] = random_dt
assert exp.metadata == exp_config[0][4]['metadata']
assert exp.pool_size is None
assert exp.max_trials is None
assert exp.version == 1
assert exp.configuration['algorithms'] == {'random': {'seed': None}}
def test_try_reset_after_race_condition(self, exp_config, new_config):
"""Cannot set a configuration after init if it looses a race condition,
but can set it if reloaded.
The experiment from process which first writes to db is initialized
properly. The experiment which looses the race condition cannot be
initialized and needs to be rebuilt.
"""
exp = Experiment(new_config['name'])
# Another experiment gets configured first
experiment_count_before = count_experiment(exp)
naughty_little_exp = Experiment(new_config['name'])
naughty_little_exp.configure(new_config)
assert naughty_little_exp._init_done is True
assert exp._init_done is False
assert (experiment_count_before + 1) == count_experiment(exp)
# First experiment won't be able to be configured
with pytest.raises(DuplicateKeyError) as exc_info:
exp.configure(new_config)
assert 'duplicate key error' in str(exc_info.value)
# Still not more experiment in DB
assert (experiment_count_before + 1) == count_experiment(exp)
# Retry configuring the experiment
new_config['metadata']['datetime'] = naughty_little_exp.metadata['datetime']
exp = Experiment(new_config['name'])
exp.configure(new_config)
def test_try_set_after_race_condition(self, exp_config, new_config):
"""Cannot set a configuration after init if it looses a race
condition.
The experiment from process which first writes to db is initialized
properly. The experiment which looses the race condition cannot be
initialized and needs to be rebuilt.
"""
exp = Experiment(new_config['name'])
assert exp.id is None
# Another experiment gets configured first
experiment_count_before = count_experiment(exp)
naughty_little_exp = Experiment(new_config['name'])
assert naughty_little_exp.id is None
naughty_little_exp.configure(new_config)
assert naughty_little_exp._init_done is True
assert exp._init_done is False
assert (experiment_count_before + 1) == count_experiment(exp)
# First experiment won't be able to be configured
with pytest.raises(DuplicateKeyError) as exc_info:
exp.configure(new_config)
assert 'duplicate key error' in str(exc_info.value)
def test_existing_experiment(self, create_db_instance, exp_config):
"""Hit exp_name + user's name in the db, fetch most recent entry."""
exp = Experiment('supernaedo2-dendi')
assert exp._init_done is False
assert_protocol(exp, create_db_instance)
assert exp._id == exp_config[0][0]['_id']
assert exp.name == exp_config[0][0]['name']
assert exp.refers == exp_config[0][0]['refers']
assert exp.metadata == exp_config[0][0]['metadata']
assert exp.pool_size == exp_config[0][0]['pool_size']
assert exp.max_trials == exp_config[0][0]['max_trials']
assert exp.algorithms == exp_config[0][0]['algorithms']
assert exp.working_dir == exp_config[0][0]['working_dir']
assert exp.version == 1
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
def test_new_experiment_due_to_name(self, create_db_instance, random_dt):
"""Hit user name, but exp_name does not hit the db, create new entry."""
exp = Experiment('supernaekei')
assert exp._init_done is False
assert_protocol(exp, create_db_instance)
assert exp._id is None
assert exp.name == 'supernaekei'
assert exp.refers == {}
assert exp.metadata['user'] == 'tsirif'
assert len(exp.metadata) == 1
assert exp.pool_size is None
assert exp.max_trials is None
assert exp.algorithms is None
assert exp.working_dir is None
assert exp.version == 1
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
Note that if we would raise RaceCondition, the conflict would still occur since
the version number fetched will not be the new one from the resolution but the requested
one. Therefore raising and handling RaceCondition would lead to infinite recursion in
the experiment builder.
"""
user_args = ['--x~normal(0,1)']
metadata = dict(user='tsirif', datetime=datetime.datetime.utcnow(), user_args=user_args)
algorithms = {'random': {'seed': None}}
config = dict(name='experiment_test', metadata=metadata, version=1, algorithms=algorithms)
backward.populate_priors(config['metadata'])
get_storage().create_experiment(config)
parent_id = config.pop('_id')
looser = Experiment('experiment_test', version=1)
# Simulate exp2 winning the race condition
config2 = copy.deepcopy(config)
config2['version'] = 2
config2['metadata']['user_args'].append("--y~+normal(0,1)")
backward.populate_priors(config2['metadata'])
config2['refers'] = dict(parent_id=parent_id, root_id=parent_id, adapters=[])
get_storage().create_experiment(config2)
# Now exp3 losses the race condition
config3 = copy.deepcopy(config)
config3['metadata']['user_args'].pop()
config3['metadata']['user_args'].append("--z~+normal(0,1)")
backward.populate_priors(config3['metadata'])
config3['version'] = 1
def test_experiment_with_parent(self, create_db_instance, random_dt, exp_config):
"""Configure an existing experiment with parent."""
exp = Experiment('supernaedo2.1')
exp.algorithms = {'random': {'seed': None}}
exp.configure(exp.configuration)
assert exp._init_done is True
assert_protocol(exp, create_db_instance)
assert exp._id is not None
assert exp.name == 'supernaedo2.1'
assert exp.configuration['refers'] == exp_config[0][4]['refers']
assert exp.metadata == exp_config[0][4]['metadata']
assert exp.pool_size == 2
assert exp.max_trials == 1000
assert exp.version == 1
assert exp.configuration['algorithms'] == {'random': {'seed': None}}
def test_old_experiment_w_version_bigger_than_max(self, create_db_instance,
parent_version_config, child_version_config):
"""Create an already existing experiment with a too large version."""
create_db_instance.write('experiments', parent_version_config)
create_db_instance.write('experiments', child_version_config)
exp = Experiment("old_experiment", user="corneauf", version=8)
assert exp.version == 2
version: int
Version of the experiment.
space: dict or Space object
Optimization space of the algorithm. If dict, should have the form
`dict(name='(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Parallel strategy to use to parallelize the algorithm.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
storage: dict, optional
Configuration of the storage backend.
"""
experiment = Experiment(name=name, version=version)
experiment._id = kwargs.get('_id', None) # pylint:disable=protected-access
experiment.pool_size = kwargs.get('pool_size')
if experiment.pool_size is None:
experiment.pool_size = orion.core.config.experiment.get(
'pool_size', deprecated='ignore')
experiment.max_trials = kwargs.get('max_trials', orion.core.config.experiment.max_trials)
experiment.space = _instantiate_space(space)
experiment.algorithms = _instantiate_algo(experiment.space, kwargs.get('algorithms'))
experiment.producer = kwargs.get('producer', {})
experiment.producer['strategy'] = _instantiate_strategy(experiment.producer.get('strategy'))
experiment.working_dir = kwargs.get('working_dir', orion.core.config.experiment.working_dir)
experiment.metadata = kwargs.get('metadata', {'user': kwargs.get('user', getpass.getuser())})
experiment.refers = kwargs.get('refers', {'parent_id': None, 'root_id': None, 'adapter': []})
experiment.refers['adapter'] = _instantiate_adapters(experiment.refers.get('adapter', []))
return experiment