Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
with open(os.path.join(comb_directory, 'test_data.yaml'), 'w') as f:
f.write(yaml.safe_dump(test_data, default_flow_style=False, allow_unicode=True))
print "Starting new combination in %s" % comb_directory
# Decide statistics location
stats_filename = os.path.join(comb_directory, str(seed) + ".csv")
log_filename = os.path.join(comb_directory, str(seed) + "_vebose.log")
# Check whether these stats have already been produced...
if os.path.isfile(stats_filename):
print "%s found, skipping." % stats_filename
continue
# ... or are being produced right now
lock_filename = stats_filename + ".lock"
lock_file = open(lock_filename, "a")
try:
portalocker.lock(lock_file, portalocker.LOCK_EX | portalocker.LOCK_NB)
except portalocker.LockException:
print "%s being produced, skipping." % stats_filename
lock_file.close()
continue;
if precmd:
seeded_precmd = precmd.replace("$seed", str(seed))
print "Running pre-command %s" % seeded_precmd
subprocess.call(seeded_precmd, shell=True, cwd=new_directory)
print "Running MicroGP to produce %s..." % stats_filename
stats_filename_temp = str(seed) + time.strftime("_%Y-%m-%d,%H:%M:%S") + ".csv"
if os.path.isfile(os.path.join(new_directory, stats_filename_temp)):
os.remove(os.path.join(new_directory, stats_filename_temp))
process = subprocess.Popen([
os.path.abspath(ugp),
'--randomSeed', str(seed),
'--log', 'verbose.log', 'verbose', 'brief',
def __init__(self, name):
self.file = open(name + ".lock", "w")
try:
portalocker.lock(self.file, portalocker.LOCK_EX | portalocker.LOCK_NB)
except IOError:
raise Exception("lock " + name + " already taken, is another process already running?")
close_file = True
return False
finally:
# check for rollover
if self.max_size and new_size > self.max_size:
close_file = True
if close_file:
self._close_file(out)
if not is_new:
self.fh_cache.pop(dir_key, None)
elif is_new:
if os.name != 'nt':
portalocker.lock(out, portalocker.LOCK_EX | portalocker.LOCK_NB)
self.fh_cache[dir_key] = (out, filename)
def _get_flags(self):
if self.exclusive:
return portalocker.LOCK_NB | portalocker.LOCK_EX
else:
return portalocker.LOCK_NB | portalocker.LOCK_SH
def __enter__(self) -> Optional[GpuDeviceType]:
for gpu_id in self.candidates:
lockfile_path = os.path.join(self.lock_dir, "sockeye.gpu{}.lock".format(gpu_id))
try:
lock_file = open(lockfile_path, 'w')
except IOError:
if errno.EACCES:
logger.warning("GPU {} is currently locked by a different process "
"(Permission denied).".format(gpu_id))
continue
try:
# exclusive non-blocking lock
portalocker.lock(lock_file, portalocker.LOCK_EX | portalocker.LOCK_NB)
# got the lock, let's write our PID into it:
lock_file.write("%d\n" % os.getpid())
lock_file.flush()
self._acquired_lock = True
self.gpu_id = gpu_id
self.lock_file = lock_file
self.lockfile_path = lockfile_path
logger.info("Acquired GPU {}.".format(gpu_id))
return gpu_id
except portalocker.LockException as e:
# portalocker packages the original exception,
# we dig it out and raise if unrelated to us
if e.args[0].errno != errno.EAGAIN: # pylint: disable=no-member
def _acquire_impl(self, _flags=LOCK_EX | LOCK_NB, tmp_dir=gettempdir(), _utcnow=datetime.utcnow, _has_debug=has_debug):
current = current_thread()
self.tmp_file_name = os.path.join(tmp_dir, 'zato-lock-{}'.format(self.pub_id))
self.tmp_file = open(self.tmp_file_name, 'w+b')
pid = os.getpid()
current_name = current.name
current_ident = current.ident
contents = self.lock_template.format(
pid, current_name, current_ident, _utcnow().isoformat(), self.os_user_name,
)
self.tmp_file.write(contents.encode('utf8'))
self.tmp_file.flush()
waitTotal = 0
while ( self.checkValidLock() ):
if ( wait is None ):
raise Exception( 'lock is busy' )
else:
if ( wait != 0 ):
waitTotal += waitInterval
if ( self.debug ):
print( 'waitTotal: %d wait: %d waitInterval: %d' % ( waitTotal, wait, waitInterval ) )
if ( waitTotal > wait ):
raise Exception( 'exceeded max wait time on the lock' )
time.sleep( waitInterval )
# don't want blocking on acquired locks - even with the loop, there is still a possibility of stolen lock and exception here
self.handle = file( self.lockfile, 'w' )
portalocker.lock( self.handle, portalocker.LOCK_EX | portalocker.LOCK_NB )
if ( self.debug ):
print( 'acquired lock %s' % self.lockfile )
pickle.dump( os.getpid(), self.handle )
if ( expire is None ):
expire_time = None
else:
expire_time = datetime.datetime.now()
expire_time += datetime.timedelta( seconds = expire )
pickle.dump( expire_time, self.handle )
pickle.dump( self.lockinfo, self.handle )
self.handle.flush()