Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def get_pair_statistics(self):
"""Count frequency of all symbol pairs, and create index"""
# Data structure of pair frequencies
stats = Counter()
# Index from pairs to words
indices = defaultdict(lambda: Counter())
for i, (word, freq) in enumerate(self.vocab):
for prev, curr in pairwise(word):
stats[prev, curr] += freq
indices[prev, curr][i] += 1
return stats, indices