Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
self._timestamp = init_time
self._time = formatted_time
# In case the user provided a git repository path, a git commit is performed
# and the environment's hexsha is taken from the commit if the commit was triggered by
# this particular environment, otherwise a new one is generated
if self._git_repository is not None:
new_commit, self._hexsha = make_git_commit(self, self._git_repository,
self._git_message, self._git_fail)
# Identifier hexsha
else:
new_commit = False
if not new_commit:
# Otherwise we need to create a novel hexsha
self._hexsha = hashlib.sha1(compat.tobytes(self.v_trajectory.v_name +
str(self.v_trajectory.v_timestamp) +
str(self.v_timestamp) +
VERSION)).hexdigest()
# Create the name of the environment
short_hexsha = self._hexsha[0:7]
name = 'environment'
self._name = name + '_' + str(short_hexsha) + '_' + self._time # Name of environment
# The trajectory should know the hexsha of the current environment.
# Thus, for all runs, one can identify by which environment they were run.
self._traj._environment_hexsha = self._hexsha
self._traj._environment_name = self._name
self._logging_manager.trajectory = self._traj
self._logging_manager.remove_null_handler()
def _load(self, load_dict):
"""Loads data from `load_dict`
Reconstruction of sparse matrices similar to the :class:`~pypet.parameter.SparseParameter`.
"""
for key in compat.listkeys(load_dict):
# We delete keys over time:
if key in load_dict:
if SparseResult.IDENTIFIER in key:
new_key = key.split(SparseResult.IDENTIFIER)[0]
is_dia = load_dict.pop(new_key + SparseResult.IDENTIFIER + 'is_dia')
name_list = SparseParameter._get_name_list(is_dia)
rename_list = ['%s%s%s' % (new_key, SparseResult.IDENTIFIER, name)
for name in name_list]
data_list = [load_dict.pop(name) for name in rename_list]
matrix = SparseParameter._reconstruct_matrix(data_list)
self._data[new_key] = matrix
else:
self._data[key] = load_dict[key]
item = args[0]
try:
name = item.v_full_name
instance = item
create_new = False
except AttributeError:
pass
# If the item is not an instance yet, check if args[0] is a class and args[1] is
# a string describing the new name of the instance.
# If args[0] is not a class it is assumed to be the name of the new instance.
if create_new:
if len(args) > 0 and inspect.isclass(args[0]):
constructor = args.pop(0)
if len(args) > 0 and isinstance(args[0], compat.base_type):
name = args.pop(0)
elif 'name' in kwargs:
name = kwargs.pop('name')
elif 'full_name' in kwargs:
name = kwargs.pop('full_name')
else:
raise ValueError('Could not determine a name of the new item you want to add. '
'Either pass the name as positional argument or as a keyword '
'argument `name`.')
if check_naming:
split_names = name.split('.')
for idx, name in enumerate(split_names):
translated_shortcut = self._translate_shortcut(name)
if translated_shortcut:
translated_shortcut = self._replace_wildcard(translated_shortcut)
def _remove_subtree_inner(node, predicate):
if not predicate(node):
return False
elif node.v_is_group:
for name_ in itools.chain(compat.listkeys(node._leaves),
compat.listkeys(node._groups)):
child_ = node._children[name_]
child_deleted = _remove_subtree_inner(child_, predicate)
if child_deleted:
_delete_from_children(node, name_)
del child_
for link_ in compat.listkeys(node._links):
node.f_remove_link(link_)
if len(node._children) == 0:
self._delete_node(node)
return True
else:
return False
else:
self._delete_node(node)
self.report_progress = (5, 'pypet', logging.INFO)
elif isinstance(self.report_progress, (int, float)):
self.report_progress = (self.report_progress, 'pypet', logging.INFO)
elif isinstance(self.report_progress, compat.base_type):
self.report_progress = (5, self.report_progress, logging.INFO)
elif len(self.report_progress) == 2:
self.report_progress = (self.report_progress[0], self.report_progress[1],
logging.INFO)
if self.log_config:
if self.log_config == pypetconstants.DEFAULT_LOGGING:
pypet_path = os.path.abspath(os.path.dirname(__file__))
init_path = os.path.join(pypet_path, 'logging')
self.log_config = os.path.join(init_path, 'default.ini')
if isinstance(self.log_config, compat.base_type):
if not os.path.isfile(self.log_config):
raise ValueError('Could not find the logger init file '
'`%s`.' % self.log_config)
parser = NoInterpolationParser()
parser.read(self.log_config)
elif isinstance(self.log_config, cp.RawConfigParser):
parser = self.log_config
else:
parser = None
if parser is not None:
self._sp_config = self._parser_to_string_io(parser)
self._mp_config = self._find_multiproc_options(parser)
if self._mp_config is not None:
self._mp_config = self._parser_to_string_io(self._mp_config)
def _estimate_memory_utilization(self, process_dict):
"""Estimates memory utilization to come if process was started"""
n_processes = len(process_dict)
total_utilization = psutil.virtual_memory().percent
sum = 0.0
for proc in compat.itervalues(process_dict):
try:
sum += psutil.Process(proc.pid).memory_percent()
except (psutil.NoSuchProcess, ZeroDivisionError):
pass
curr_all_processes = sum
missing_utilization = max(0.0, n_processes * self._est_per_process - curr_all_processes)
estimated_utilization = total_utilization
estimated_utilization += missing_utilization
estimated_utilization += self._est_per_process
return estimated_utilization
result_list.append(candidate)
full_name_set.add(candidate.v_full_name)
elif shortcuts:
candidate_set = set(candidate_split_name)
climbing = True
for name in split_name:
if name not in candidate_set:
climbing = False
break
if climbing:
count = 0
candidate_length = len(candidate_split_name)
for idx in compat.xrange(candidate_length):
if idx + split_length - count > candidate_length:
break
if split_name[count] == candidate_split_name[idx]:
count += 1
if count == len(split_name):
result_list.append(candidate)
full_name_set.add(candidate.v_full_name)
break
return result_list
String constant specifying if we want to store, load or remove.
The corresponding constants are defined at the top of this module.
:param name: String name of item to store, load or remove.
:param args: Additional arguments passed to the storage service
:param kwargs: Additional keyword arguments passed to the storage service
:return:
A formatted request that can be handled by the storage service, aka
a tuple: (msg, item_to_store_load_or_remove, args, kwargs)
"""
if not isinstance(name, compat.base_type):
raise TypeError('No string!')
node = self._root_instance.f_get(name)
return self._fetch_from_node(store_load, node, args, kwargs)
self._runfunc = None
self._args = ()
self._kwargs = {}
self._postproc = None
self._postproc_args = ()
self._postproc_kwargs = {}
self._immediate_postproc = immediate_postproc
self._user_pipeline = False
self._git_repository = git_repository
self._git_message = git_message
self._git_fail = git_fail
# Check if a novel trajectory needs to be created.
if isinstance(trajectory, compat.base_type):
# Create a new trajectory
self._traj = Trajectory(trajectory,
add_time=add_time,
dynamic_imports=dynamic_imports,
wildcard_functions=wildcard_functions,
comment=comment)
self._timestamp = self.v_trajectory.v_timestamp # Timestamp of creation
self._time = self.v_trajectory.v_time # Formatted timestamp
else:
self._traj = trajectory
# If no new trajectory is created the time of the environment differs
# from the trajectory and must be computed from the current time.
init_time = time.time()
formatted_time = datetime.datetime.fromtimestamp(init_time).strftime(
'%Y_%m_%d_%Hh%Mm%Ss')
def f_add_link(self, name_or_item, full_name_or_item=None):
"""Adds a link to an existing node.
Can be called as ``node.f_add_link(other_node)`` this will add a link the `other_node`
with the link name as the name of the node.
Or can be called as ``node.f_add_link(name, other_node)`` to add a link to the
`other_node` and the given `name` of the link.
In contrast to addition of groups and leaves, colon separated names
are **not** allowed, i.e. ``node.f_add_link('mygroup.mylink', other_node)``
does not work.
"""
if isinstance(name_or_item, compat.base_type):
name = name_or_item
if isinstance(full_name_or_item, compat.base_type):
instance = self.v_root.f_get(full_name_or_item)
else:
instance = full_name_or_item
else:
instance = name_or_item
name = instance.v_name
return self._nn_interface._add_generic(self, type_name=LINK,
group_type_name=GROUP, args=(name, instance),
kwargs={},
add_prefix=False)