$! CHECK_PMAS_QUEUES.COM	Check PMDF PreciseMail processing queues
$!
$! This command procedure checks the MAIL$BATCH_PMAS_% queues for jobs which
$! were submitted over an hour ago, and deletes them.  The reason for doing
$! this is that the PreciseMail channel program for PMDF has a bug which
$! causes it to "hang" when it encounters certain problems connecting to
$! Process Software.
$!
$! Ideally we'd like to kill such jobs after 1 hour of elapsed time, but there's
$! no easy way to do this so we assume that all jobs start within a few  minutes
$! of being submitted (which is usually the case).
$!
$!	Created 1-Nov-2007 by Jeremy Begg, VSM Software Services Pty Ltd.
$!
$! Modifications
$! -------------
$! 11-Jan-2008	J.Begg		Set up new SLIDER, different PMAS queue names.
$!				Send an email alert each time we delete one or
$!				more PMAS jobs.
$!
$! 26-Feb-2008	J.Begg		Check MAIL$BATCH too.
$!
$!******************************************************************************
$
$	set noon
$
$	node		= f$getsyi("NODENAME")
$	reqd_privs	= "OPER,SYSPRV,CMKRNL,SYSLCK"
$	say		= "write sys$output"
$	last_hour	= ""
$	start_date	= f$cvtime(,,"DATE")
$	timestamp	= "TRUE"
$	tmp_file	= f$fao("SYS$SCRATCH:JOB_DETAIL.!SL", f$getjpi(0,"PID"))
$	tmp_file_open	= "FALSE"
$
$	old_privs = f$setprv(reqd_privs)
$	if .not. f$privilege(reqd_privs)
$	then
$	     say "%CHECK-F-NOPRIV, need ", reqd_privs, " to run this procedure"
$	     exit 36
$	endif
$
$!
$!******************************************************************************
$START:
$!
$! Use F$GETQUI to work our way through the PMAS jobs
$!
$	one_hour_ago = f$cvtime("-1:0:0",,)
$	job_delete_list = ""
$	if f$cvtime(,,"HOUR") .nes. last_hour then $ timestamp = "TRUE"
$	if timestamp then $ say f$fao("!%D  Checking all PMDF_PMAS_* queues for jobs submitted before !AS ...", 0, f$cvtime("-1:0:0","ABSOLUTE",))
$	job_count = 0
$
$Queue_Loop:
$	    queue = f$getqui("DISPLAY_QUEUE","QUEUE_NAME","MAIL$BATCH_PMAS_%")
$	    if queue .eqs. "" then $ goto NEXT_LOOP
$	    queue_stopped = f$getqui("DISPLAY_QUEUE","QUEUE_STOPPED",queue)
$	    queue_jobs = f$getqui("DISPLAY_QUEUE","EXECUTING_JOB_COUNT",queue,"WILDCARD")
$	    if timestamp
$	    then
$		if queue_stopped
$		then
$		    say f$time(), "  queue ", queue, " is stopped!"
$		else if queue_jobs .gt. 0
$		then
$		    say f$fao("!%D  queue !AS has !SL job!%S executing", 0, queue, queue_jobs)
$		else
$		    say f$time(), "  queue ", queue, " is idle"
$		endif
$		endif
$	    endif
$	    if queue_jobs .eq. 0 then $ goto Queue_Loop
$Job_Loop:
$		job_entry = f$integer(f$getqui("DISPLAY_JOB","ENTRY_NUMBER",,"ALL_JOBS"))
$		if job_entry .eq. 0 then $ goto Queue_Loop
$		if .not. f$getqui("DISPLAY_JOB","JOB_EXECUTING",,"FREEZE_CONTEXT") then $ goto Job_Loop
$		job_submit_time = f$getqui("DISPLAY_JOB","SUBMISSION_TIME",,"FREEZE_CONTEXT")
$		if timestamp then $ say f$fao("!%D    entry !SL submitted at !AS", 0, job_entry, job_submit_time)
$		if f$cvtime(job_submit_time,,) .les. one_hour_ago
$		then
$		    ! This job has been here too long
$		    if .not. tmp_file_open
$		    then
$			create 'tmp_file'
$			open/append tmp_file 'tmp_file'
$			tmp_file_open = "TRUE"
$			write tmp_file "PreciseMail job(s) to be delete at ", f$extract(0,20,f$time())
$			write tmp_file f$fao("!80*-")
$		    endif
$		    job_detail = f$fao("Job !AS, entry !SL in queue !AS submitted at !AS", -
				f$getqui("DISPLAY_JOB","JOB_NAME",,"FREEZE_CONTEXT"), -
				job_entry, queue, job_submit_time)
$		    say f$time(), "  ", job_detail
$		    write tmp_file job_detail
$		    job_delete_list = f$fao("!AS,!SL", job_delete_list, job_entry)
$		    job_count = job_count + 1
$		endif
$		goto Job_Loop
$
$!
$!
$NEXT_LOOP:
$!
$! If any jobs have to be deleted, do it now
$!
$	if tmp_file_open then $ close tmp_file
$	tmp_file_open = "FALSE"
$	t_file = f$search(tmp_file)
$	if t_file .eqs. "" then $ t_file = "NL:"
$	if f$length(job_delete_list) .gt. 0
$	then
$	    job_delete_list = job_delete_list - ","
$	    say f$time(), "  Deleting job(s) ..."
$	    delete/entry=('job_delete_list')
$	    pmdf send/subj="''f$fao("CHECK_PMAS_QUEUES deleted !SL job!%S!!", job_count)'" 't_file' postmaster
$	endif
$	if t_file .nes. "NL:" then $ delete/nolog 'tmp_file';*
$
$CHECK_MAIL$BATCH:
$!
$! Check that MAIL$BATCH is running and that the Delivery job has run
$! recently
$!
$	if f$getqui("DISPLAY_QUEUE","QUEUE_STOPPED","MAIL$BATCH")
$	then
$	    say f$time(), "  MAIL$BATCH is stopped!"
$	    show queue/full/all mail$batch/out='tmp_file'
$	    mailx/subject="MAIL$BATCH is STOPPED on ''node'!!" 'tmp_file' flash::jeremy
$	    delete/nolog 'tmp_file';*
$	    goto CHECK_MAIL$BATCH_End
$	endif
$
$	mbatch_pending = f$getqui("DISPLAY_QUEUE","PENDING_JOB_COUNT","MAIL$BATCH","ALL_JOBS,WILDCARD")
$!
$! Look for the "PMDF Delivery" job
$!
$CHECK_MAIL$BATCH_Job:
$	    mbatch_jobname = f$getqui("DISPLAY_JOB","JOB_NAME",,"ALL_JOBS")
$	    if mbatch_jobname .eqs. "PMDF delivery"
$	    then
$		if f$getqui("DISPLAY_JOB","JOB_PENDING",,"FREEZE_CONTEXT")
$		then
$		    say f$fao("!%D  PMDF delivery job is waiting to run! (!SL pending job!%S)", 0, mbatch_pending)
$		    show queue/full/all/out='tmp_file' mail$batch
$		    mailx/subject="PMDF delivery job is blocked on ''node'!!" 'tmp_file' flash::jeremy
$		    delete/nolog 'tmp_file';*
$		endif
$		goto CHECK_MAIL$BATCH_End
$	    else if mbatch_jobname .eqs. ""
$	    then
$		! PMDF Delivery job is missing, submit it now
$		say f$time(), "  PMDF delivery job is missing!"
$		show queue/full/all/out='tmp_file' mail$batch
$		mailx/subject="PMDF delivery job missing on ''node', resubmitting now" 'tmp_file' flash::jeremy
$		delete/nolog 'tmp_file';*
$		@pmdf_com:pmdf_submit_jobs
$		goto CHECK_MAIL$BATCH_End
$	    else
$		goto CHECK_MAIL$BATCH_Job
$	    endif
$	    endif
$
$CHECK_MAIL$BATCH_End:
$	xxx = f$getqui("CANCEL_OPERATION")
$
$
$! Prepare for the next loop.  We run continuously until midnight,
$! at which time JOBSCHED will submit a new monitoring job.
$!
$	last_hour = f$cvtime(,,"HOUR")
$	timestamp = "FALSE"
$	wait 00:05:00
$	if f$cvtime(,,"DATE") .eqs. start_date then $ goto START
$
$!
$! We've been here all day, time to die
$!
$	purge/keep=7 madm_logs:check_pmas_queues.log
$	exit