How to use the turbinia.jobs.manager function in turbinia

To help you get started, we’ve selected a few turbinia examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github google / turbinia / turbinia / client.py View on Github external
config.LoadConfig()
    psq_publisher = pubsub.PublisherClient()
    psq_subscriber = pubsub.SubscriberClient()
    datastore_client = datastore.Client(project=config.TURBINIA_PROJECT)
    try:
      self.psq = psq.Queue(
          psq_publisher, psq_subscriber, config.TURBINIA_PROJECT,
          name=config.PSQ_TOPIC, storage=psq.DatastoreStorage(datastore_client))
    except exceptions.GoogleCloudError as e:
      msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
      log.error(msg)
      raise TurbiniaException(msg)

    # Deregister jobs from blacklist/whitelist.
    disabled_jobs = list(config.DISABLED_JOBS) if config.DISABLED_JOBS else []
    job_manager.JobsManager.DeregisterJobs(jobs_blacklist, jobs_whitelist)
    if disabled_jobs:
      log.info(
          'Disabling jobs that were configured to be disabled in the '
          'config file: {0:s}'.format(', '.join(disabled_jobs)))
      job_manager.JobsManager.DeregisterJobs(jobs_blacklist=disabled_jobs)

    # Check for valid dependencies/directories.
    check_dependencies(config.DEPENDENCIES)
    check_directory(config.MOUNT_DIR_PREFIX)
    check_directory(config.OUTPUT_DIR)
    check_directory(config.TMP_DIR)

    log.info('Starting PSQ listener on queue {0:s}'.format(self.psq.name))
    self.worker = psq.Worker(queue=self.psq)
github google / turbinia / turbinia / task_manager.py View on Github external
Raises:
      TurbiniaException: When no Jobs are found.
    """
    if not self.jobs:
      raise turbinia.TurbiniaException(
          'Jobs must be registered before evidence can be added')
    log.info('Adding new evidence: {0:s}'.format(str(evidence_)))
    job_count = 0
    jobs_whitelist = evidence_.config.get('jobs_whitelist', [])
    jobs_blacklist = evidence_.config.get('jobs_blacklist', [])
    if jobs_blacklist or jobs_whitelist:
      log.info(
          'Filtering Jobs with whitelist {0!s} and blacklist {1!s}'.format(
              jobs_whitelist, jobs_blacklist))
      jobs_list = jobs_manager.JobsManager.FilterJobObjects(
          self.jobs, jobs_blacklist, jobs_whitelist)
    else:
      jobs_list = self.jobs

    # TODO(aarontp): Add some kind of loop detection in here so that jobs can
    # register for Evidence(), or or other evidence types that may be a super
    # class of the output of the job itself.  Short term we could potentially
    # have a run time check for this upon Job instantiation to prevent it.
    for job in jobs_list:
      # Doing a strict type check here for now until we can get the above
      # comment figured out.
      # pylint: disable=unidiomatic-typecheck
      if [True for t in job.evidence_input if type(evidence_) == t]:
        job_instance = job(
            request_id=evidence_.request_id, evidence_config=evidence_.config)
        self.running_jobs.append(job_instance)
github google / turbinia / turbinia / workers / __init__.py View on Github external
log.debug('Task {0:s} {1:s} awaiting execution'.format(self.name, self.id))
    evidence = evidence_decode(evidence)
    with filelock.FileLock(config.LOCK_FILE):
      log.info('Starting Task {0:s} {1:s}'.format(self.name, self.id))
      original_result_id = None
      try:
        self.result = self.setup(evidence)
        original_result_id = self.result.id
        evidence.validate()

        # TODO(wyassine): refactor it so the result task does not
        # have to go through the preprocess stage. At the moment
        # self.results.setup is required to be called to set its status.
        # Check if Task's job is available for the worker.
        active_jobs = list(job_manager.JobsManager.GetJobNames())
        if self.job_name.lower() not in active_jobs:
          message = (
              'Task will not run due to the job: {0:s} being disabled '
              'on the worker.'.format(self.job_name))
          self.result.log(message, level=logging.ERROR)
          self.result.status = message
          return self.result.serialize()

        if self.turbinia_version != turbinia.__version__:
          message = (
              'Worker and Server versions do not match: {0:s} != {1:s}'.format(
                  self.turbinia_version, turbinia.__version__))
          self.result.log(message, level=logging.ERROR)
          self.result.status = message
          return self.result.serialize()
github google / turbinia / turbinia / task_manager.py View on Github external
def generate_request_finalize_tasks(self, job):
    """Generates the Tasks to finalize the given request ID.

    Args:
      job (TurbiniaJob): The last Job that was run for this request.
    """
    request_id = job.request_id
    final_job = jobs_manager.JobsManager.GetJobInstance('FinalizeRequestJob')
    final_job.request_id = request_id
    final_job.evidence.config = job.evidence.config
    log.debug(
        'Request {0:s} done, but not finalized, creating FinalizeRequestJob '
        '{1:s}'.format(request_id, final_job.id))

    # Finalize tasks use EvidenceCollection with all evidence created by the
    # request or job.
    final_evidence = evidence.EvidenceCollection()
    final_evidence.request_id = request_id
    self.running_jobs.append(final_job)

    # Gather evidence created by every Job in the request.
    for running_job in self.running_jobs:
      if running_job.request_id == request_id:
        final_evidence.collection.extend(running_job.evidence.collection)
github google / turbinia / turbinia / jobs / sshd.py View on Github external
"""Create task.

    Args:
      evidence: List of evidence objects to process

    Returns:
        A list of tasks to schedule.
    """
    tasks = []
    for evidence_item in evidence:
      if evidence_item.artifact_name == 'SshdConfigFile':
        tasks.append(sshd.SSHDAnalysisTask())
    return tasks


manager.JobsManager.RegisterJobs([SSHDAnalysisJob, SSHDExtractionJob])
github google / turbinia / turbinia / jobs / tomcat.py View on Github external
Args:
      evidence: List of evidence objects to process

     Returns:
        A list of tasks to schedule.
    """
    tasks = []

    for evidence_item in evidence:
      if evidence_item.artifact_name == 'TomcatFile':
        tasks.append(tomcat.TomcatAnalysisTask())
    return tasks


manager.JobsManager.RegisterJobs([TomcatExtractionJob, TomcatAnalysisJob])