Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
with ProcessPoolExecutor(max_workers=self.n_workers) as executor:
futures = [
executor.submit(
sb.corpus_bleu, b[0], b[1], force=True,
tokenize=tokenizer
)
for b in batches
]
progress = as_completed(futures)
if self.verbose:
progress = tqdm(progress)
for future in progress:
s = future.result()
ref_len += s.ref_len
sys_len += s.sys_len
for n in range(sb.NGRAM_ORDER):
correct[n] += s.counts[n]
total[n] += s.totals[n]
corpus_score = sb.compute_bleu(
correct, total, sys_len, ref_len, smooth_method='exp'
).bp
return corpus_score
def score_corpus_multiprocess(
self, hypothesis: List[str], references: List[List[str]]
) -> float:
tokenizer = get_optional_dict(self.extra_args, 'bleu_tokenizer', 'none')
if self.n_workers == 1:
corpus_score = sb.corpus_bleu(
hypothesis, references, force=True, tokenize=tokenizer
).score
else:
batches = list(
self._batch(hypothesis, references, n_batches=self.n_workers)
)
ref_len, sys_len = 0, 0
correct = [0 for _ in range(sb.NGRAM_ORDER)]
total = [0 for _ in range(sb.NGRAM_ORDER)]
with ProcessPoolExecutor(max_workers=self.n_workers) as executor:
futures = [
executor.submit(
sb.corpus_bleu, b[0], b[1], force=True,
tokenize=tokenizer
)
for b in batches
]
progress = as_completed(futures)
if self.verbose:
progress = tqdm(progress)
for future in progress:
s = future.result()
ref_len += s.ref_len
sys_len += s.sys_len
for n in range(sb.NGRAM_ORDER):