Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@workers.app.task(name=SchedulerCeleryTasks.TENSORBOARDS_SCHEDULE_DELETION, ignore_result=True)
def tensorboards_schedule_deletion(tensorboard_job_id, immediate=False):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id,
include_deleted=True)
if not tensorboard:
return None
tensorboard.archive()
if tensorboard.is_stoppable:
project = tensorboard.project
workers.send(
SchedulerCeleryTasks.TENSORBOARDS_STOP,
kwargs={
'project_name': project.unique_name,
'project_uuid': project.uuid.hex,
'tensorboard_job_name': tensorboard.unique_name,
def _run(task_bind, *args, **kwargs):
experiment_id = kwargs['experiment_id']
experiment = get_valid_experiment(experiment_id=experiment_id)
if not experiment:
raise OperationRunError(
'The Experiment `{}` does not exist anymore.'.format(experiment_id))
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_BUILD,
kwargs={'experiment_id': experiment_id},
countdown=conf.get('GLOBAL_COUNTDOWN'))
def perform_destroy(self, instance):
instance.archive()
workers.send(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_SCHEDULE_DELETION,
kwargs={'notebook_job_id': instance.id, 'immediate': True})
def projects_schedule_deletion(project_id, immediate=False):
project = get_valid_project(project_id=project_id, include_deleted=True)
if not project:
# No need to check this project
return
project.archive()
message = 'Project is scheduled for deletion.'
groups = project.all_experiment_groups.exclude(
status__status__in=ExperimentGroupLifeCycle.DONE_STATUS).distinct()
for group in groups.values_list('id', flat=True):
workers.send(
SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP,
kwargs={
'experiment_group_id': group,
'collect_logs': False,
'message': message,
})
experiments = project.all_experiments.exclude(
experiment_group__isnull=True,
status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct().iterator()
for experiment in experiments:
if experiment.is_stoppable:
workers.send(
SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={
'project_name': experiment.project.unique_name,
'project_uuid': experiment.project.uuid.hex,
@workers.app.task(name=SchedulerCeleryTasks.DELETE_ARCHIVED_PROJECT, ignore_result=True)
def delete_archived_project(project_id):
try:
Project.archived.get(id=project_id).delete()
except Project.DoesNotExist:
pass
def _handle_build_job_cleaned_triggered(cls, event: 'Event') -> None:
instance = event.instance
if not instance.is_managed:
return
if not instance or not instance.has_specification or not instance.is_stoppable:
return
workers.send(
SchedulerCeleryTasks.BUILD_JOBS_STOP,
kwargs={
'project_name': instance.project.unique_name,
'project_uuid': instance.project.uuid.hex,
'build_job_name': instance.unique_name,
'build_job_uuid': instance.uuid.hex,
'update_status': False,
'collect_logs': False,
'is_managed': instance.is_managed,
})
def _handle_experiment_post_run(cls, event: 'Event') -> None:
instance = event.instance
if not instance or not instance.has_specification or not instance.jobs.count() > 0:
return
# Schedule stop for this experiment because other jobs may be still running
group = instance.experiment_group
workers.send(
SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={
'project_name': instance.project.unique_name,
'project_uuid': instance.project.uuid.hex,
'experiment_name': instance.unique_name,
'experiment_uuid': instance.uuid.hex,
'experiment_group_name': group.unique_name if group else None,
'experiment_group_uuid': group.uuid.hex if group else None,
'specification': instance.content,
'update_status': False,
'collect_logs': True,
'is_managed': instance.is_managed,
},
countdown=RedisTTL.get_for_experiment(experiment_id=instance.id))
def delete_archived_jobs() -> None:
last_date = get_date_check(days=conf.get(CLEANING_INTERVALS_ARCHIVES))
ids = Job.archived.filter(
# We only check values that will not be deleted by the archived projects
project__deleted=False,
updated_at__lte=last_date).values_list('id', flat=True)
for _id in ids:
workers.send(
SchedulerCeleryTasks.DELETE_ARCHIVED_JOB,
kwargs={'job_id': _id})
def post(self, request, *args, **kwargs):
auditor.record(event_type=JOB_STOPPED_TRIGGERED,
instance=self.job,
actor_id=request.user.id,
actor_name=request.user.username)
workers.send(
SchedulerCeleryTasks.JOBS_STOP,
kwargs={
'project_name': self.project.unique_name,
'project_uuid': self.project.uuid.hex,
'job_name': self.job.unique_name,
'job_uuid': self.job.uuid.hex,
'update_status': True,
'collect_logs': True,
'is_managed': self.job.is_managed,
})
return Response(status=status.HTTP_200_OK)
def _handle_tensorboard_post_run(cls, event: 'Event') -> None:
instance = event.instance
if not instance or not instance.has_specification:
return
workers.send(
SchedulerCeleryTasks.TENSORBOARDS_STOP,
kwargs={
'project_name': instance.project.unique_name,
'project_uuid': instance.project.uuid.hex,
'tensorboard_job_name': instance.unique_name,
'tensorboard_job_uuid': instance.uuid.hex,
'update_status': False,
'collect_logs': True,
'is_managed': instance.is_managed,
})