Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def expire_all_unassigned_hits(self):
"""
Move through the whole hit_id list and attempt to expire the HITs, though this
only immediately expires those that aren't assigned.
"""
# TODO note and mark assigned hits as ones to be expired later
shared_utils.print_and_log(
logging.INFO,
'Expiring all unassigned HITs...',
should_print=not self.is_test,
)
completed_ids = self.worker_manager.get_complete_hits()
for hit_id in self.hit_id_list:
if hit_id not in completed_ids:
# TODO get confirmation that the HIT is acutally expired
mturk_utils.expire_hit(self.is_sandbox, hit_id)
def _log_missing_agent(self, worker_id, assignment_id):
"""Logs when an agent was expected to exist, yet for some reason it
didn't. If these happen often there is a problem"""
shared_utils.print_and_log(
logging.WARN,
'Expected to have an agent for {}_{}, yet none was found'.format(
worker_id, assignment_id
),
def _on_socket_dead(self, worker_id, assignment_id):
"""Handle a disconnect event, update state as required and notifying
other agents if the disconnected agent was in conversation with them
returns False if the socket death should be ignored and the socket
should stay open and not be considered disconnected
"""
agent = self.worker_manager._get_agent(worker_id, assignment_id)
if agent is None:
# This worker never registered, so we don't do anything
return
shared_utils.print_and_log(
logging.DEBUG,
'Worker {} disconnected from {} in status {}'.format(
worker_id, agent.conversation_id, agent.get_status()
),
)
if agent.get_status() == AssignState.STATUS_NONE:
# Agent never made it to onboarding, delete
agent.set_status(AssignState.STATUS_DISCONNECT)
agent.reduce_state()
elif agent.get_status() == AssignState.STATUS_ONBOARDING:
# Agent never made it to task pool, the onboarding thread will die
# and delete the agent if we mark it as a disconnect
agent.set_status(AssignState.STATUS_DISCONNECT)
agent.reduce_state()
agent.disconnected = True
tmp_dir=self.opt['tmp_dir'],
)
else:
self.populate_task_files(task_directory_path)
self.server_url = server_utils.setup_server(
self.server_task_name,
self.task_files_to_copy,
self.opt['local'],
heroku_team,
self.opt['hobby'],
tmp_dir=self.opt['tmp_dir'],
)
shared_utils.print_and_log(logging.INFO, self.server_url)
shared_utils.print_and_log(
logging.INFO, "MTurk server setup done.\n", should_print=True
)
self.task_state = self.STATE_SERVER_ALIVE
def block_worker(self, reason='unspecified'):
"""Block a worker from our tasks"""
self.mturk_manager.block_worker(worker_id=self.worker_id, reason=reason)
shared_utils.print_and_log(
logging.WARN,
'Blocked worker ID: {}. Reason: {}'.format(self.worker_id, reason),
should_print=True,
)
self.opt['hit_description'], self.task_group_id
),
hit_keywords=self.opt['hit_keywords'],
hit_reward=self.opt['reward'],
# Set to 30 minutes by default
assignment_duration_in_seconds=self.opt.get(
'assignment_duration_in_seconds', 30 * 60
),
is_sandbox=self.opt['is_sandbox'],
qualifications=qualifications,
auto_approve_delay=self.auto_approve_delay,
)
mturk_chat_url = '{}/chat_index?task_group_id={}'.format(
self.server_url, self.task_group_id
)
shared_utils.print_and_log(logging.INFO, mturk_chat_url)
mturk_page_url = None
if self.topic_arn is not None:
mturk_utils.subscribe_to_hits(hit_type_id, self.is_sandbox, self.topic_arn)
for _i in range(num_hits):
mturk_page_url, hit_id, mturk_response = mturk_utils.create_hit_with_hit_type(
opt=self.opt,
page_url=mturk_chat_url,
hit_type_id=hit_type_id,
num_assignments=1,
is_sandbox=self.is_sandbox,
)
if self.db_logger is not None:
self.db_logger.log_hit_status(mturk_response)
self.hit_id_list.append(hit_id)
def _send_packet(self, packet, connection_id, send_time):
"""
Sends a packet, blocks if the packet is blocking.
"""
# Send the packet
pkt = packet.as_dict()
if pkt['data'] is None or packet.status == Packet.STATUS_ACK:
return # This packet was _just_ acked.
shared_utils.print_and_log(logging.DEBUG, 'Send packet: {}'.format(packet))
result = self._safe_send(
json.dumps({'type': data_model.SOCKET_ROUTE_PACKET_STRING, 'content': pkt})
)
if not result:
# The channel died mid-send, wait for it to come back up
self._safe_put(connection_id, (send_time, packet))
return
if packet.status != Packet.STATUS_ACK:
packet.status = Packet.STATUS_SENT
# Handles acks and blocking
if packet.requires_ack:
if packet.blocking:
# Put the packet right back into its place to prevent sending
def _on_socket_dead(self, worker_id, assignment_id):
"""
Handle a disconnect event, update state as required and notifying other agents
if the disconnected agent was in conversation with them.
returns False if the socket death should be ignored and the socket should stay
open and not be considered disconnected
"""
agent = self.worker_manager._get_agent(worker_id, assignment_id)
if agent is None:
# This worker never registered, so we don't do anything
return
shared_utils.print_and_log(
logging.DEBUG,
'Worker {} disconnected from {} in status {}'.format(
worker_id, agent.conversation_id, agent.get_status()
),
)
if agent.get_status() == AssignState.STATUS_NONE:
# Agent never made it to onboarding, delete
agent.set_status(AssignState.STATUS_DISCONNECT)
agent.reduce_state()
elif agent.get_status() == AssignState.STATUS_ONBOARDING:
# Agent never made it to task pool, the onboarding thread will die
# and delete the agent if we mark it as a disconnect
agent.set_status(AssignState.STATUS_DISCONNECT)
agent.reduce_state()
agent.disconnected = True
'be an agent in ParlAI.\nDuring this process, Internet connection '
'is required, and you should turn off your computer\'s auto-sleep '
'feature.',
should_print=True,
)
if self.opt['max_connections'] == 0:
shared_utils.print_and_log(
logging.INFO,
'Enough HITs will be created to fulfill {} times the '
'number of conversations requested, extra HITs will be expired'
' once the desired conversations {}.'
''.format(self.hit_mult, fin_word),
should_print=True,
)
else:
shared_utils.print_and_log(
logging.INFO,
'Enough HITs will be launched over time '
'up to a max of {} times the amount requested until the '
'desired number of conversations {}.'
''.format(self.hit_mult, fin_word),
should_print=True,
)
input('Please press Enter to continue... ')
shared_utils.print_and_log(logging.NOTSET, '', True)
if self.opt['local'] is True:
shared_utils.print_and_log(
logging.INFO,
"In order to run the server locally, you will need "
"to have a public HTTPS endpoint (SSL signed) running on "
"the server you are currently excecuting ParlAI on. Enter "
def approve_work(self, assignment_id, override_rejection=False):
"""approve work for a given assignment through the mturk client"""
client = mturk_utils.get_mturk_client(self.is_sandbox)
client.approve_assignment(
AssignmentId=assignment_id, OverrideRejection=override_rejection
)
if self.db_logger is not None:
self.db_logger.log_approve_assignment(assignment_id)
shared_utils.print_and_log(
logging.INFO, 'Assignment {} approved.' ''.format(assignment_id)
)