Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
filters,
1 # procs_cpu_bound
),
)
proc.daemon = False
proc.start()
procs.append(proc)
receive_pipes.append(receive_pipe)
logger.debug(f"len(procs): {len(bounders)}")
for i in range(len(procs)):
logger.debug(f"Waiting pipe number: {i}")
df_received = receive_pipes[i].recv()
if df is None:
df = df_received
else:
df = pd.concat(objs=[df, df_received], ignore_index=True)
logger.debug(f"Waiting proc number: {i}")
procs[i].join()
logger.debug(f"Closing proc number: {i}")
receive_pipes[i].close()
return df
columns=columns,
filters=filters,
procs_cpu_bound=procs_cpu_bound)
else:
df = Pandas._read_parquet_path(session_primitives=session_primitives,
path=path[0],
columns=columns,
filters=filters,
procs_cpu_bound=procs_cpu_bound)
for p in path[1:]:
df_aux = Pandas._read_parquet_path(session_primitives=session_primitives,
path=p,
columns=columns,
filters=filters,
procs_cpu_bound=procs_cpu_bound)
df = pd.concat(objs=[df, df_aux], ignore_index=True)
return df