Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __parse_in_multiprocess(self):
"""Method to parse data in bulk by parsing each data item
against each template and saving results in results list.
"""
log.info("ttp.parse: parse using multiple processes")
num_processes = cpu_count()
for template in self._templates:
num_jobs = 0
tasks = JoinableQueue()
results_queue = Queue()
workers = [
_worker(
tasks,
results_queue,
lookups=template.lookups,
vars=template.vars,
groups=template.groups,
macro_text=template.macro_text,
)
for i in range(num_processes)
]
[w.start() for w in workers]
for input_name, input_obj in template.inputs.items():
for datum in input_obj.data:
task_dict = {
"data": datum,
"groups_indexes": input_obj.groups_indexes,