Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def make_predictions(conf, shot_list, loader):
loader.set_inference_mode(True)
use_cores = max(1, mp.cpu_count()-2)
if backend == 'tf' or backend == 'tensorflow':
first_time = "tensorflow" not in sys.modules
if first_time:
import tensorflow as tf
os.environ['KERAS_BACKEND'] = 'tensorflow'
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto(device_count={"CPU": use_cores})
set_session(tf.Session(config=config))
else:
os.environ['THEANO_FLAGS'] = 'device=cpu'
# import theano
from plasma.models.builder import ModelBuilder
specific_builder = ModelBuilder(conf)
else:
pool = None
if ocp is None:
raise NotImplementedError('\"ocp\" must be defined.')
"""
Main code
"""
f_ocp = FuncOCP(ocp)
# breakpoint()
# s_ocp = SymOCP(ocp)
# breakpoint()
# bvp = FuncBVP(s_bvp)
logging.debug('Using ' + str(n_cpus) + '/' + str(pathos.multiprocessing.cpu_count()) + ' CPUs. ')
if bvp is None:
bvp, ocp_map, ocp_map_inverse = ocp2bvp(ocp, method=method, optim_options=optim_options)
logging.debug('Resulting BVP problem:')
for key in bvp.raw.keys():
logging.debug(str(key) + ': ' + str(bvp.raw[key]))
else:
if ocp_map is None or ocp_map_inverse is None:
raise ValueError('BVP problem must have an associated \'ocp_map\' and \'ocp_map_inverse\'')
solinit = Trajectory()
solinit.const = np.array(bvp.raw['constants_values'])
def __init__(self,process):
self.size = mp.cpu_count()
self.process = process
self.phase=None
self.pool = mp.ProcessingPool(self.size)
shot_list = ShotList()
shot_list.load_from_shot_list_files_objects(shot_files, all_signals)
shot_list_picked = shot_list.random_sublist(use_shots)
previously_saved, machines_saved = self.previously_saved_stats()
machines_to_compute = all_machines - machines_saved
recompute = conf['data']['recompute_normalization']
if recompute:
machines_to_compute = all_machines
previously_saved = False
if not previously_saved or len(machines_to_compute) > 0:
if previously_saved:
self.load_stats(verbose=True)
print('computing normalization for machines {}'.format(
machines_to_compute))
use_cores = max(1, mp.cpu_count()-2)
pool = mp.Pool(use_cores)
print('running in parallel on {} processes'.format(
pool._processes))
start_time = time.time()
for (i, stats) in enumerate(pool.imap_unordered(
self.train_on_single_shot, shot_list_picked)):
# for (i,stats) in
# enumerate(map(self.train_on_single_shot,shot_list_picked)):
if stats.machine in machines_to_compute:
self.incorporate_stats(stats)
self.machines.add(stats.machine)
sys.stdout.write('\r'
+ '{}/{}'.format(i, len(shot_list_picked)))
pool.close()
pool.join()
def _check_threads(self, message_end=None):
"""
Checks number of threads
:param message_end: closing part of the error message.
"""
if PATHOS_FOUND:
threads = AbinsModules.AbinsParameters.performance['threads']
if not (isinstance(threads, six.integer_types) and 1 <= threads <= mp.cpu_count()):
raise RuntimeError("Invalid number of threads for parallelisation over atoms" + message_end)
def _check_threads(self, message_end=None):
"""
Checks number of threads
:param message_end: closing part of the error message.
"""
if PATHOS_FOUND:
atoms_threads = AbinsParameters.atoms_threads
if not (isinstance(atoms_threads, (int, long)) and 1 <= atoms_threads <= mp.cpu_count()):
raise RuntimeError("Invalid number of threads for parallelisation over atoms" + message_end)
q_threads = AbinsParameters.q_threads
if not (isinstance(q_threads, (int, long)) and 1 <= q_threads <= mp.cpu_count()):
raise RuntimeError("Invalid number of threads for parallelisation over q" + message_end)
if atoms_threads * q_threads > mp.cpu_count():
raise RuntimeError("User asked for more threads than available.")
def _calc_num_threads(self, df_size: int, query_size: int, max_threads=None) -> int:
num_queries = df_size * query_size
if mp.cpu_count() == 1:
max = 1
else:
max = mp.cpu_count() - 1
calc = int(num_queries / 5000)
if calc > max:
r = max
elif calc <= 1:
if num_queries > 1000:
r = 2
else:
r = 1
else:
r = calc
if max_threads is not None and r > max_threads:
return max_threads
return r
def _calc_num_threads(self, df_size: int, query_size: int, max_threads=None) -> int:
num_queries = df_size * query_size
if mp.cpu_count() == 1:
max = 1
else:
max = mp.cpu_count() - 1
calc = int(num_queries / 5000)
if calc > max:
r = max
elif calc <= 1:
if num_queries > 1000:
r = 2
else:
r = 1
else:
r = calc
if max_threads is not None and r > max_threads:
return max_threads
return r