Документ взят из кэша поисковой машины. Адрес оригинального документа : http://www.stsci.edu/spst/UnixTransition/doc/submit_codine.py Дата изменения: Fri Apr 8 12:46:13 2016 Дата индексирования: Mon Apr 11 01:34:56 2016 Кодировка: Поисковые слова: fire

#
#MODULE submit_codine
#
#***********************************************************************
"""

**PURPOSE** --
A low-level routine for submitting periodic batch jobs to the
Sun Grid Engine (SGE).

**DEVELOPER** --
Don Chance

**MODIFICATION HISTORY** --
Initial implementation 7/31/01
Modified to support non-periodic submissions. drc 11/12/01
Raise an error when the resubmission doesn't work. drc 2/25/02
Change spss_sys_util.command to os.system when resubmitting the batch
job. drc 3/6/02
Mods to provide a lot more information if qsub fails. mdr 3/7/02
Mod so next execution time is localtime of machine running job. mdr 8/24/10
Use time rather than time_util to handle machine localtime. mdr 9/7/10
Enhance -period, add cron option add a submit-only option mdr 10/7/10
Change to switches for each cron time field mdr 10/8/10
Update for Python 3 compatiblity and verision 3.0 of APScheduler. drc 9/29/14
More updates for operating on Linux with newer APScheduler version drc 1/7/15
"""
#***********************************************************************
import time
import spss_sys_util
import os
import datetime
import pytz
try:
# for APScheduler versions less than 3.0
from apscheduler.triggers import CronTrigger
cronversion = 1
except:
# for APScheduler 3.0
from apscheduler.triggers.cron import CronTrigger
cronversion = 3

__version__ = "15.01.07"


TZ = pytz.timezone('US/Eastern')


def run(*args):
"""Submit a periodic batch job to Codine.

do submit_codine -period [-sts | -ets]
-cron [[-year ][-month ][-day ][-week ]
[-day_of_week ][-hour ][-minute ]
[-second [-subonly]
[-dir ]
-qsub ...
-tool ...

Note: The argument names (period, cron, qsub, tool, dir, sts, ste, subonly)
MUST NOT be abbreviated, as the full names are important for distinguishing
between submit_codine's arguments and those for qsub and the tool.

The argument -tool is required and may only appear after -qsub.
Any data after the -tool is assumed to be the tool name and its
arguments.

All arguments between -qsub and -tool are assumed to be arguments
for the SGE qsub command.

-period specifies the time (in seconds) to leave between
this execution and the beginning of the next. If "-period 0",
then the batch job executes only once.
-sts indicates the -period should be applied from job start to
job start.
-ets indicates the -period should be applied from the job end to
job start (default).
Note: -sts and -ets are mutually exclusive.

-cron uses a cron-like time specification as to when the job should
fire.
The following switches allow for various time specified firings
(any unspecified switch will default to fire on every value unless
noted otherwise).
A cron with no time specification at all will be a job which will
fire every minute on the minute.

-year = 4-digit year number
-month = month number (1-12)
-day = day of the month (1-31)
-week = ISO week number (1-53)
-day_of_week = number or name of weekday (0-6 or mon,tue,wed,thu,fri,sat,sun)
-hour = hour (0-23)
-minute = minute (0-59) (default = 0)

The following values/expressions are allowed in each of the above -cron time switches:
Exp Field Description
a-b any Fire on any value within the a-b range (a must be smaller than b)
a-b/c any Fire every c values within the a-b range
xth_y day Fire on the x-th occurrence of weekday y within the month
last_x day Fire on the last occurrence of weekday x within the month
x,y,z any Fire on any matching expression; can combine any number of any of the above expressions

Note: -period and -cron are mutually exclusive.

-subonly specifies to only submit the job. This allows for use of
submit_codine.py from within a batch job to run a tool as a
separate batch job rather than inline.

-dir specifies the working directory to use when executing the
tool. The working directory defaults to the current working
directory from which submit_codine is executed.
"""
if not args:
print(run.__doc__)
return spss_sys_util.SUCCESS

jobsttime = time.time()
#cronsttime = datetime.datetime.today()
print("Current time: " + time.strftime("%X %x %Z", time.localtime(jobsttime)))

args = list(args)
# Parse the arguments
# Note because of submit_codines more complicated argument structure
# spst_getopt was not used.

# Deal with -period/-cron error cases
if (('-period' in args) and ('-cron' in args)):
raise ValueError("-period and -cron may not both be specified.")
# end if

if (('-period' not in args) and ('-cron' not in args)):
raise ValueError("Either -period or -cron must be specified.")
# end if

# Parse out everything for dealing with a -period job
if '-period' in args:
job_type = '-period'
period_index = args.index('-period')
waittime = float(args[period_index+1])
# end if

# Default if both -sts and -ets are missing from the commandline
period_type = '-ets'

# Error cases
if (('-sts' in args) and ('-ets' in args)):
raise ValueError("-sts and -ets may not both be specified.")
# endif
if ((('-sts' in args) and not ('-period' in args)) or
(('-ets' in args) and not ('-period' in args))):
raise ValueError("-period is required if either -sts or -ets is specified.")
# end if

# Valid cases
if ('-sts' in args):
period_type = '-sts'
# end if
if ('-ets' in args):
period_type = '-ets'
# end if

# Parse out everything for dealing with a -cron job
if ('-cron' in args):
job_type = '-cron'
#cron_index = args.index('-cron')
cronspec = []
if ('-year' in args):
year_index = args.index('-year')
cyr = args[year_index+1]
cronspec.append('-year')
cronspec.append(cyr)
else:
cyr = "*"
# endif
if ('-month' in args):
month_index = args.index('-month')
cmn = args[month_index+1]
cronspec.append('-month')
cronspec.append(cmn)
else:
cmn = "*"
# end if
if ('-day' in args):
day_index = args.index('-day')
tcdy = args[day_index+1]
cdy = tcdy.replace("_", " ")
cronspec.append('-day')
cronspec.append(tcdy)
else:
cdy = "*"
# end if
if ('-week' in args):
week_index = args.index('-week')
cwk = args[week_index+1]
cronspec.append('-week')
cronspec.append(cwk)
else:
cwk = "*"
# end if
if ('-day_of_week' in args):
dow_index = args.index('-day_of_week')
cdow = args[dow_index+1]
cronspec.append('-day_of_week')
cronspec.append(cdow)
else:
cdow = "*"
# end if
if ('-hour' in args):
hour_index = args.index('-hour')
chour = args[hour_index+1]
cronspec.append('-hour')
cronspec.append(chour)
else:
chour = "*"
# end if
if ('-minute' in args):
minute_index = args.index('-minute')
cmin = args[minute_index+1]
cronspec.append('-minute')
cronspec.append(cmin)
else:
cmin = "0"
# end if
csec = "0"
crontime = CronTrigger(year=cyr, month=cmn, day=cdy, day_of_week=cdow, hour=chour, minute=cmin, second=csec)
# end if

if ('-subonly' in args):
submit_only = 1
else:
submit_only = 0
# end if

if '-qsub' in args:
qsub_index = args.index('-qsub')
else:
raise ValueError("-qsub argument is required.")

dir = os.getcwd()
if '-dir' in args:
dir_index = args.index('-dir')
if os.path.isdir(args[dir_index + 1]):
dir = args[dir_index + 1]
else:
dir = spss_sys_util.resolver(args[dir_index + 1])
if not dir:
raise ValueError(args[dir_index + 1] +
" is not a valid directory path")

if '-tool' in args:
tool_index = args.index('-tool')
else:
raise ValueError("-tool argument is required.")

qsub_arg_list = args[qsub_index+1:tool_index]
tool_arg_list = args[tool_index+1:]
tool_arg_string = " ".join(tool_arg_list)
tool_name = tool_arg_list[0]

# We will call the tool through do_batch_job.csh in order
# to force qsub to login rather than inherit from the current
# environment.
path = spss_sys_util.resolver('COM', 'do_batch_job.csh')
if ((spss_sys_util.mode() != 'interactive') and (not submit_only)):
# Running from a batch job and no -subonly switch
# Not the first iteration...

# Move to the working directory...
print("Executing os.chdir: " + dir)
os.chdir(dir)

print("Executing do " + tool_name)
dopath = spss_sys_util.resolver('COM', 'do_spss_cmds.csh')
status, text = spss_sys_util.command('%s %s' % (dopath,
tool_arg_string))
print("Status = " + str(status))
print(text)

jobendtime = time.time()
#cronendtime = datetime.datetime.today()
cronendtime = datetime.datetime.now(TZ)

# Compute the next execution time for the job
if (job_type == '-period'):
# Exit if the waittime = 0
if int(waittime) <= 0:
return spss_sys_util.SUCCESS
# end if
if (period_type == '-sts'):
job_relative_time = jobsttime
else:
job_relative_time = jobendtime
# end if
next_exec_time = job_relative_time + float(waittime)
next_fire_time = time.strftime('%Y%m%d%H%M.%S', time.localtime(next_exec_time))
print("Next execution time: " + time.strftime("%X %x %Z", time.localtime(next_exec_time)))
elif (job_type == '-cron'):
if cronversion == 1:
next_cron_time = crontime.get_next_fire_time(cronendtime)
else:
next_cron_time = crontime.get_next_fire_time(cronendtime, cronendtime)
next_fire_time = next_cron_time.strftime("%Y%m%d%H%M.%S")
# Since the datetime usage here is TZ-naive, spit it out via time
print("Next execution time: " + next_cron_time.strftime("%X %x") + " " + time.strftime("%Z"))
else:
raise ValueError("Unknown job_type %s" % job_type)
# end if

# Find and remove the value of the -a option from the qsub argument
# list. In its place, put the next execution time
if '-a' in qsub_arg_list:
i = qsub_arg_list.index('-a')
else:
# No -a option, so we'll add it...
qsub_arg_list.append('-a')
i = qsub_arg_list.index('-a')

qsub_arg_list = (qsub_arg_list[:i+1] +
[next_fire_time] +
qsub_arg_list[i+2:])

# Check the qsub arguments for -cwd. If it's not found, add it.
if '-cwd' in qsub_arg_list:
qsub_arg_list.index('-cwd')
else:
qsub_arg_list.append('-cwd')

qsub_arg_string = " ".join(qsub_arg_list)
if (job_type == '-period'):
job_type_extras = "%i %s" % (waittime, period_type)
elif (job_type == '-cron'):
job_type_extras = " ".join(cronspec)
else:
raise ValueError("Unknown job_type %s" % job_type)
# end if
cmd = 'qsub %s %s submit_codine %s %s -dir %s -qsub %s -tool %s' % (
qsub_arg_string,
path,
job_type, job_type_extras,
dir,
qsub_arg_string,
tool_arg_string)

print("Resubmitting batchjob: " + cmd)
print("from node: " + os.environ['HOST'])

# status, text = spss_sys_util.command(cmd)
status = os.system(cmd)
print ("Status = " + str(status))
# print text
if status:
if (spss_sys_util.mode() == 'batch'):
error_text = "qsub failed\n" \
+ "QUEUE = %s\n" % os.environ['QUEUE'] \
+ "HOSTNAME = %s\n" % os.environ['HOSTNAME'] \
+ "HOST = %s\n" % os.environ['HOST'] \
+ "JOB_ID = %s\n" % os.environ['JOB_ID'] \
+ "JOB_SCRIPT = %s\n" % os.environ['JOB_SCRIPT'] \
+ "command = %s\n" % cmd
else:
error_text = "qsub failed"
raise OSError(status, error_text)

return spss_sys_util.SUCCESS


if __name__ == '__main__':
import sys
if len(sys.argv) > 1:
print(sys.argv)
run(*tuple(sys.argv[1:]))
else:
run()