Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def worker(client_id, func_name, df, key, subset):
worker = gear.Worker(client_id)
worker.addServer(host=ct.GEARMAND_HOST, port=ct.GEARMAND_PORT)
worker.registerFunction(func_name)
redis = create_redis_obj()
while True:
job = worker.getJob()
info = json.loads(job.arguments.decode('utf-8'))
tmp_df = pd.DataFrame(info, index=[0])
tmp_redis = redis.get(key)
if tmp_redis is not None: df = _pickle.loads(tmp_redis)
df = df.append(tmp_df)
df = df.drop_duplicates(subset)
redis.set(key, _pickle.dumps(df, 2))
job.sendWorkComplete()