Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def train(data_train, data_eval, model):
"""Training function."""
# backend specific implementation
param_dict = model.bert.collect_params()
if backend == 'horovod':
hvd.broadcast_parameters(param_dict, root_rank=0)
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
logging.info('Creating distributed trainer...')
lr = args.lr
optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
if args.dtype == 'float16':
optim_params['multi_precision'] = True
dynamic_loss_scale = args.dtype == 'float16'
if dynamic_loss_scale:
loss_scale_param = {'scale_window': 2000 / num_workers, 'init_scale': 2**10}
else:
loss_scale_param = None
def evaluate(data_eval, model, ctx, log_interval, dtype):
"""Evaluation function."""
logging.info('Running evaluation ... ')
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
eval_begin_time = time.time()
begin_time = time.time()
step_num = 0
running_mlm_loss = running_nsp_loss = 0
total_mlm_loss = total_nsp_loss = 0
running_num_tks = 0
for _, data_batch in enumerate(data_eval):
step_num += 1
data_list = split_and_load(data_batch, ctx)
ns_label_list, ns_pred_list = [], []
mask_label_list, mask_pred_list, mask_weight_list = [], [], []
for data in data_list:
def train(data_train, model, nsp_loss, mlm_loss, vocab_size, ctx, store):
"""Training function."""
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
lr = args.lr
optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
if args.dtype == 'float16':
optim_params['multi_precision'] = True
trainer = mx.gluon.Trainer(model.collect_params(), 'bertadam', optim_params,
update_on_kvstore=False, kvstore=store)
dynamic_loss_scale = args.dtype == 'float16'
fp16_trainer = FP16Trainer(trainer, dynamic_loss_scale=dynamic_loss_scale)
if args.start_step:
state_path = os.path.join(args.ckpt_dir, '%07d.states.%02d'%(args.start_step, 0))
logging.info('Loading trainer state from %s', state_path)
def train(data_train, data_eval, model):
"""Training function."""
# backend specific implementation
param_dict = model.bert.collect_params()
if backend == 'horovod':
hvd.broadcast_parameters(param_dict, root_rank=0)
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
logging.info('Creating distributed trainer...')
lr = args.lr
optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
if args.dtype == 'float16':
optim_params['multi_precision'] = True
dynamic_loss_scale = args.dtype == 'float16'
if dynamic_loss_scale:
loss_scale_param = {'scale_window': 2000 / num_workers, 'init_scale': 2**10}
else:
loss_scale_param = None
# backend specific implementation
def evaluate(data_eval, model, nsp_loss, mlm_loss, vocab_size, ctx, log_interval, dtype):
"""Evaluation function."""
logging.info('Running evaluation ... ')
mlm_metric = MaskedAccuracy()
nsp_metric = MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
eval_begin_time = time.time()
begin_time = time.time()
step_num = 0
running_mlm_loss = running_nsp_loss = 0
total_mlm_loss = total_nsp_loss = 0
running_num_tks = 0
for _, dataloader in enumerate(data_eval):
for _, data_batch in enumerate(dataloader):
step_num += 1
data_list = split_and_load(data_batch, ctx)
loss_list = []
def train(data_train, data_eval, model, nsp_loss, mlm_loss, vocab_size, ctx):
"""Training function."""
hvd.broadcast_parameters(model.collect_params(), root_rank=0)
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
logging.debug('Creating distributed trainer...')
lr = args.lr
optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
if args.dtype == 'float16':
optim_params['multi_precision'] = True
dynamic_loss_scale = args.dtype == 'float16'
if dynamic_loss_scale:
loss_scale_param = {'scale_window': 2000 / num_workers}
else:
loss_scale_param = None
trainer = hvd.DistributedTrainer(model.collect_params(), 'bertadam', optim_params)
fp16_trainer = FP16Trainer(trainer, dynamic_loss_scale=dynamic_loss_scale,
def train(data_train, model, nsp_loss, mlm_loss, vocab_size, ctx, store):
"""Training function."""
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
lr = args.lr
optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
if args.dtype == 'float16':
optim_params['multi_precision'] = True
trainer = mx.gluon.Trainer(model.collect_params(), 'bertadam', optim_params,
update_on_kvstore=False, kvstore=store)
dynamic_loss_scale = args.dtype == 'float16'
fp16_trainer = FP16Trainer(trainer, dynamic_loss_scale=dynamic_loss_scale)
if args.start_step:
state_path = os.path.join(args.ckpt_dir, '%07d.states.%02d'%(args.start_step, 0))
def evaluate(data_eval, model, nsp_loss, mlm_loss, vocab_size, ctx, log_interval, dtype):
"""Evaluation function."""
logging.info('Running evaluation ... ')
mlm_metric = MaskedAccuracy()
nsp_metric = MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
eval_begin_time = time.time()
begin_time = time.time()
step_num = 0
running_mlm_loss = running_nsp_loss = 0
total_mlm_loss = total_nsp_loss = 0
running_num_tks = 0
for _, dataloader in enumerate(data_eval):
for _, data_batch in enumerate(dataloader):
step_num += 1
data_list = split_and_load(data_batch, ctx)
loss_list = []
ns_label_list, ns_pred_list = [], []
def evaluate(data_eval, model, ctx, log_interval, dtype):
"""Evaluation function."""
logging.info('Running evaluation ... ')
mlm_metric = nlp.metric.MaskedAccuracy()
nsp_metric = nlp.metric.MaskedAccuracy()
mlm_metric.reset()
nsp_metric.reset()
eval_begin_time = time.time()
begin_time = time.time()
step_num = 0
running_mlm_loss = running_nsp_loss = 0
total_mlm_loss = total_nsp_loss = 0
running_num_tks = 0
for _, data_batch in enumerate(data_eval):
step_num += 1
data_list = split_and_load(data_batch, ctx)
ns_label_list, ns_pred_list = [], []
mask_label_list, mask_pred_list, mask_weight_list = [], [], []