Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
information about themselves, before being put into a conversation.
You can end the conversation by sending a message ending with
`[DONE]` from human_1.
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id]
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = MTurkMultiAgentDialogOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
# You can set onboard_function to None to skip onboarding
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
# add additional model args
opt['override'] = {
'no_cuda': True,
'interactive_mode': True,
'tensorboard_log': False
}
# Set the task name to be the folder name
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
# append the contents of task_config.py to the configuration
opt.update(task_config)
mturk_agent_id = 'Tourist'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.ready_to_accept_workers()
mturk_manager.create_hits([LOCALE_QUALIF_SDBOX])
mturk_manager.set_onboard_function(onboard_function=None)
# mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
help='How many passages to retrieve per dialog \
message',
)
opt = argparser.parse_args()
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
if 'data_path' not in opt:
opt['data_path'] = os.getcwd() + '/data/' + opt['task']
opt['current_working_dir'] = os.getcwd()
opt.update(task_config)
mturk_agent_ids = [APPRENTICE, WIZARD]
opt['min_messages'] = 2
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
setup_personas_with_wiki_links(opt)
ir_agent, task = setup_retriever(opt)
persona_generator = PersonasGenerator(opt)
wiki_title_to_passage = setup_title_to_passage(opt)
mturk_manager.setup_server(task_directory_path=directory_path)
worker_roles = {}
connect_counter = AttrDict(value=0)
try:
mturk_manager.start_new_run()
if not opt['is_sandbox']:
with open(os.path.join(opt['current_working_dir'], 'mtdont.txt')) as f:
lines = [l.replace('\n', '') for l in f.readlines()]
for w in lines:
mturk_manager.soft_block_worker(w)
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id])
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = ModelEvaluatorOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
def main():
completed_workers = []
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id])
mturk_manager.setup_server()
qual_name = 'ParlAIExcludeQual{}t{}'.format(
random.randint(10000, 99999), random.randint(10000, 99999)
)
qual_desc = (
'Qualification for a worker not correctly completing the '
'first iteration of a task. Used to filter to different task pools.'
)
qualification_id = mturk_utils.find_or_create_qualification(
qual_name, qual_desc, opt['is_sandbox']
)
print('Created qualification: ', qualification_id)
def run_onboard(worker):
world = QualificationFlowOnboardWorld(opt, worker)
while not world.episode_done():
else:
folder_name = '{}-{}'.format(start_opt['model'], start_time)
start_opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
if 'data_path' not in start_opt:
start_opt['data_path'] = os.path.join(
os.getcwd(), 'data', 'wizard_eval', folder_name
)
start_opt.update(task_config)
if not start_opt.get('human_eval'):
mturk_agent_ids = ['PERSON_1']
else:
mturk_agent_ids = ['PERSON_1', 'PERSON_2']
mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids)
topics_generator = TopicsGenerator(start_opt)
directory_path = os.path.dirname(os.path.abspath(__file__))
mturk_manager.setup_server(task_directory_path=directory_path)
worker_roles = {}
connect_counter = AttrDict(value=0)
try:
mturk_manager.start_new_run()
agent_qualifications = []
if not start_opt['is_sandbox']:
# assign qualifications
if start_opt['only_masters']:
agent_qualifications.append(MASTER_QUALIF)
if start_opt['unique_workers']:
qual_name = 'UniqueChatEval'
Send an email to a worker through the mturk client.
"""
client = mturk_utils.get_mturk_client(self.is_sandbox)
response = client.notify_workers(
Subject=subject, MessageText=message_text, WorkerIds=[worker_id]
)
if len(response['NotifyWorkersFailureStatuses']) > 0:
failure_message = response['NotifyWorkersFailureStatuses'][0]
return {'failure': failure_message['NotifyWorkersFailureMessage']}
else:
return {'success': True}
# TODO consolidate base functionality out of this class and above into a
# base_crowd_manager and then expand out from there.
class StaticMTurkManager(MTurkManager):
"""
Manages interactions between MTurk agents and tasks, the task launching workflow,
and more, but only for tasks that require just 2 connections to the server: an
initial task request and the submission of results.
"""
def __init__(self, opt, is_test=False):
"""
No interaction means only ever one agent, so that's what we get.
"""
opt['max_connections'] = 0 # Max connections doesn't make sense here
opt['count_complete'] = True # No other way to count static HITs
opt['frontend_template_type'] = 'static'
super().__init__(opt, ['worker'], is_test, use_db=True)
self.hit_mult = 1 # No need to pad HITs if they're static
self.required_hits = self.num_conversations
# Set the task name to be the folder name
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
# append the contents of task_config.py to the configuration
opt.update(task_config)
# Select an agent_id that worker agents will be assigned in their world
mturk_agent_roles = ['worker_1', 'worker_2']
# Set runtime to be an hour in case workers are slow
opt['assignment_duration_in_seconds'] = 60 * 60
# Instantiate an MTurkManager with the given options and a maximum number
# of agents per world of 1 (based on the length of mturk_agent_ids)
mturk_manager = MTurkManager(
opt=opt, mturk_agent_ids=mturk_agent_roles, use_db=True
)
mturk_manager.setup_server(
task_directory_path=os.path.dirname(os.path.abspath(__file__))
)
# Create an onboard_function, which will be run for workers who have
# accepted your task and must be completed before they are put in the
# queue for a task world.
completed_agents = []
def run_onboard(worker):
nonlocal completed_agents
if worker.worker_id in completed_agents:
return
else:
help='which dialog round to show',
)
opt = argparser.parse_args()
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
if 'data_path' not in opt or opt['data_path'] == '':
opt['data_path'] = "{}/data/{}_evals".format(os.getcwd(), opt['dialog_round'])
opt['task_dir'] = os.getcwd()
if opt['dialog_round'] == 'questions':
opt.update(tc_questions)
else:
opt.update(tc_responses)
mturk_agent_ids = [RATER]
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
example_generator = IGCExampleGenerator(opt)
mturk_manager.setup_server(task_directory_path=directory_path)
try:
mturk_manager.start_new_run()
def run_onboard(worker):
worker.example_generator = example_generator
world = RoleOnboardWorld(opt, worker)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
mturk_manager.create_hits()