Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
except ImportError:
pass
args = parser.parse_args()
if not args.launchpad_file and args.config_dir and os.path.exists(os.path.join(args.config_dir,
'my_launchpad.yaml')):
args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml')
if not args.fworker_file and args.config_dir and os.path.exists(os.path.join(args.config_dir,
'my_fworker.yaml')):
args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml')
args.loglvl = 'CRITICAL' if args.silencer else args.loglvl
launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad(strm_lvl=args.loglvl)
if args.fworker_file:
fworker = FWorker.from_file(args.fworker_file)
else:
fworker = FWorker()
total_node_list = None
if args.nodefile:
if args.nodefile in os.environ:
args.nodefile = os.environ[args.nodefile]
with open(args.nodefile, 'r') as f:
total_node_list = [line.strip() for line in f.readlines()]
launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs,
args.sleep, total_node_list, args.ppn, timeout=args.timeout,
exclude_current_node=args.exclude_current_node)
parser.add_argument('--loglvl', help='level to print log messages', default='INFO')
parser.add_argument('--silencer', help='shortcut to mute log messages', action='store_true')
rapid_parser.add_argument('-q', '--njobs_queue', help='maximum jobs to keep in queue for this user', default=10, type=int)
rapid_parser.add_argument('-b', '--njobs_block', help='maximum jobs to put in a block', default=500, type=int)
rapid_parser.add_argument('--infinite', help='loop forever', action='store_true')
rapid_parser.add_argument('--sleep', help='sleep time between loops', default=60, type=int)
rapid_parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=None)
args = parser.parse_args()
rocket_params = QueueParams.from_file(args.queue_params_file)
args.loglvl = 'CRITICAL' if args.silencer else args.loglvl
if args.command == 'rapidfire':
launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else None
rapidfire(rocket_params, args.launch_dir, args.njobs_queue, args.njobs_block, args.logdir, args.loglvl, args.infinite, args.sleep, launchpad)
else:
launch_rocket_to_queue(rocket_params, args.launch_dir, args.logdir, args.loglvl)
import os, sys
from glob import glob
from dateutil import parser
from datetime import datetime
from fireworks.core.launchpad import LaunchPad
from collections import Counter
from fnmatch import fnmatch
lpdb = LaunchPad.from_file('/global/homes/m/matcomp/mp_prod/config/config_Mendel/my_launchpad.yaml')
"""
counter = Counter()
for wf_idx, wf_doc in enumerate(lpdb.workflows.find(
{'updated_on': {'$exists': 1}},
{'state': 1, 'updated_on': 1, 'nodes': 1}
)):
try:
dt = parser.parse(wf_doc['updated_on'])
except:
dt = wf_doc['updated_on']
counter['ALL_WFS'] += 1
if dt > datetime(2016, 1, 1):
counter['ALL_RECENT_WFS'] += 1
fws_fizzled = []
for fw_idx, fw_doc in enumerate(lpdb.fireworks.find(
mc = MongoClient(task_creds['host'], task_creds['port'])
db = mc[task_creds['database']]
db.authenticate(task_creds['admin_user'], task_creds['admin_password'])
tasks = db['tasks']
tasks_f = os.path.join(module_dir, 'tasks.yaml')
with open(tasks_f) as f2:
task_creds = yaml.load(f2)
mc = MongoClient(task_creds['host'], task_creds['port'])
db = mc[task_creds['database']]
db.authenticate(task_creds['admin_user'], task_creds['admin_password'])
tasks = db['tasks']
lp_f = os.path.join(module_dir, 'my_launchpad.yaml')
lpdb = LaunchPad.from_file(lp_f)
fws = lpdb.fireworks
launches = lpdb.launches
sb_f = os.path.join(module_dir, 'submission.yaml')
sbdb = SubmissionMongoAdapter.from_file(sb_f)
submissions = sbdb.jobs
bad_crystal_ids = []
crystals_file = os.path.join(module_dir, 'bad_crystals.txt')
with open(crystals_file) as f:
for line in f:
bad_crystal_ids.append(int(line.strip()))
for c_id in bad_crystal_ids:
elif not args.launchpad_file:
args.launchpad_file = LAUNCHPAD_LOC
if not args.fworker_file and os.path.exists(
os.path.join(args.config_dir, 'my_fworker.yaml')):
args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml')
elif not args.fworker_file:
args.fworker_file = FWORKER_LOC
if not args.queueadapter_file and os.path.exists(
os.path.join(args.config_dir, 'my_qadapter.yaml')):
args.queueadapter_file = os.path.join(args.config_dir, 'my_qadapter.yaml')
elif not args.queueadapter_file:
args.queueadapter_file = QUEUEADAPTER_LOC
launchpad = LaunchPad.from_file(
args.launchpad_file) if args.launchpad_file else LaunchPad(
strm_lvl=args.loglvl)
fworker = FWorker.from_file(
args.fworker_file) if args.fworker_file else FWorker()
queueadapter = load_object_from_file(args.queueadapter_file)
args.loglvl = 'CRITICAL' if args.silencer else args.loglvl
if args.command == 'rapidfire':
rapidfire(launchpad, fworker=fworker, qadapter=queueadapter, launch_dir=args.launch_dir,
nlaunches=args.nlaunches, njobs_queue=args.maxjobs_queue,
njobs_block=args.maxjobs_block, sleep_time=args.sleep,
reserve=args.reserve, strm_lvl=args.loglvl, timeout=args.timeout, fill_mode=args.fill_mode)
else:
launch_rocket_to_queue(launchpad, fworker, queueadapter,
args.launch_dir, args.reserve, args.loglvl, False, args.fill_mode, args.fw_id)
def archive_deprecated_fws():
# find all snlgroups that are deprecated, and archive all WFs that have deprecated fw_ids so we don't run them
module_dir = os.path.dirname(os.path.abspath(__file__))
snl_f = os.path.join(module_dir, 'snl.yaml')
snldb = SNLMongoAdapter.from_file(snl_f)
snlgroups = snldb.snlgroups
lp_f = os.path.join(module_dir, 'my_launchpad.yaml')
lpdb = LaunchPad.from_file(lp_f)
for g in snlgroups.find({'canonical_snl.about.remarks':'DEPRECATED'}, {'snlgroup_id': 1}):
while lpdb.fireworks.find_one({'spec.snlgroup_id': g['snlgroup_id'], 'state': {'$ne': 'ARCHIVED'}}, {'fw_id': 1}):
fw = lpdb.fireworks.find_one({'spec.snlgroup_id': g['snlgroup_id'], 'state': {'$ne': 'ARCHIVED'}}, {'fw_id': 1})
print fw['fw_id']
lpdb.archive_wf(fw['fw_id'])
print 'DONE'
kpoints = Kpoints.from_dict(relax["input"]["kpoints"])
kpoints.write_file(os.path.join(cwd, folder, "KPOINTS"))
poscar.to("POSCAR", os.path.join(cwd, folder, "POSCAR"))
tasks = [RunCustodianTask(cwd=cwd, folder=folder, debug=debug,
custodian_params=cust_params),
InsertTask(cwd=cwd, folder=folder, debug=debug,
db_credentials=db_credentials,
task_id=task)]
fw = Firework(tasks, name=folder)
fw_ids.append(fw.fw_id)
fws.append(fw)
wf = Workflow(fws, name='Workfunction Calculations')
launchpad = LaunchPad.from_file(os.path.join(os.environ["HOME"],
launchpad_dir,
"my_launchpad.yaml"))
launchpad.add_wf(wf)
@classmethod
def auto_load(cls):
s_dir = os.environ['DB_LOC']
s_file = os.path.join(s_dir, 'submission_db.yaml')
s = SubmissionHandler.Submissions.from_file(s_file)
l_dir = FWConfig().CONFIG_FILE_DIR
l_file = os.path.join(l_dir, 'my_launchpad.yaml')
lp = LaunchPad.from_file(l_file)
return SubmissionHandler(s, lp)
def get_colls():
colls = namedtuple('Collections', ['snl', 'snlgroups'])
sma = SNLMongoAdapter.from_file(snl_f)
lp = LaunchPad.from_file(fw_f)
colls.snl = sma.snl
colls.snlgroups = sma.snlgroups
colls.fireworks = lp.fireworks
colls.launches = lp.launches
with open(tasks_f) as f2:
task_creds = yaml.load(f2)
mc = MongoClient(task_creds['host'], task_creds['port'])
db = mc[task_creds['database']]
db.authenticate(task_creds['admin_user'], task_creds['admin_password'])
colls.tasks = db['tasks']
return colls