96c5ae2cdd2f6d9c369472b2d1977c02d12c652e,foreman/data_refinery_foreman/foreman/main.py,,retry_lost_processor_jobs,#,210

Before Change

def retry_lost_processor_jobs() -> None:
    Retry processor jobs who never even got started for too long.
    minimum_creation_time = timezone.now() - MAX_QUEUE_TIME
    lost_jobs = ProcessorJob.objects.filter(
        success=None,
        retried=False,
        start_time=None,
        end_time=None,
        created_at__lt=minimum_creation_time
        // TEMPORARY for Jackie"s grant
    ).exclude(pipeline_applied=ProcessorPipeline.NONE.value)

    handle_processor_jobs(lost_jobs)

After Change

    nomad_port = get_env_variable("NOMAD_PORT", "4646")
    nomad_client = nomad.Nomad(nomad_host, port=int(nomad_port), timeout=5)
    lost_jobs = []
    for job in potentially_lost_jobs:
        try:
            job_status = nomad_client.job.get_job(job.nomad_job_id)["Status"]
            // If the job is still pending, then it makes sense that it hasn"t started.
            if job_status is not "pending":
                // However if it"s not pending, then it may have
                // started since our original query.
                job.refresh_from_db()
                if job.start_time is None:
                    // Nope, this job is lost.
                    lost_jobs.append(job)
        except URLNotFoundNomadException:
            lost_jobs.append(job)

    handle_processor_jobs(lost_jobs)


def monitor_jobs():

Instances