Документ взят из кэша поисковой машины. Адрес оригинального документа : http://www.stsci.edu/spst/UnixTransition/doc/sched_snap_sus.py Дата изменения: Fri Apr 8 12:46:13 2016 Дата индексирования: Mon Apr 11 02:12:24 2016 Кодировка: Поисковые слова: http astrokuban.info astrokuban

#!/usr/bin/env python
#
#MODULE sched_snap_sus
#
#***********************************************************************
"""

**PURPOSE** --
Schedule SNAP SU's onto a cclist.

**DEVELOPER** --
Gary Bower and Merle Reinhart

**KEY** --
allowed_options = a list of legal options on the command line.
args = a list containing the arguments from the command line.
begin_time_x,y = the scheduled begin time used in cmp_begin_time.
calendar_commands = a list of commands for the calendar -report jobs.
cclist = a dummy variable for looping.
ccl_intermed = the intermediate ccl object during snap iterations.
ccl_final = the cclist to which snaps have been scheduled,
expressed as a ccl_util object.
ccl_start = the cclist to which to schedule snaps, expressed as a
ccl_util object.
ccl_tend = the end time of input_cclist.
ccl_tstart = the start time of input_cclist.
comlist = a list of containing calendar commands to do the calendar
-report jobs.
curcyc(0,1) = snap pool names for the current cycle.
current_command = a string holding the current command while being constructed.
current_cycle = the current cycle from the command line expressed as
an integer.
cur_cyc = a string denoting the delta cycle number of a snap pool.
cur_rec = an integer counter marking the current record in grecs.
cur_sus_files = a list of SU files (list).
cycle = the current proposal cycle (integer or string).
cycle_pool = a dummy variable for looping over the snap pools.
cyc_name = a string representing the cycle number of a snap pool.
dbConn = a stpydb object that connects to the database.
dict = a dummy variable for iterating over a list of dictionaries
obtained from snap_dict.
e = a dummy string needed for iterating over a list of errors.
elapsed_time = the time required for running this tool, expressed
as a time_util object.
errors = a list of logfile lines containing the case-insenstive string
'error'.
errors_list = a list of error messages.
factor = a string showing the percentage of a snap program that has
been scheduled.
file_snap_sus = a temporary file name to which snap_sus are written.
first_line = a string containing the first line in snap_temp_file.
g = a file object to which output is to be written.
gaps = the filename of the gaps file generated from ccl_final.
grecs = a list of lines in snap_temp_file.
header = removal of the first element in grecs.
i = a dummy variable for looping.
inequality = converting snap_key from alphanumeric to a mathematics
logical operator.
input_cclist = the input cclist converted to a ccl_util object.
j = a dummy variable for looping.
key = a dummy string needed for iterating over a list of keys.
line = a dummy string needed for iterating over a list.
lines = a list of lines in an input file.
niter = the integer number of snap iterations that have been started.
no_snaps = a Boolean variable indicating if there are no snaps that
can be scheduled.
node = a dummy string needed for looping over the nodes list.
num_pools = the integer number of pools provided in snap_temp_file.
This is currently set to 4, but may need to be changed if there is
a change in policy regarding snap programs from previous cycles.
options = a dictionary containing the options and values in args.
outfile = an output file object.
output_file_dict = a dictionary of calendar -report output files,
sorted by snap pool.
output_root = the root name of the output log and report files.
ow = a file object to which output is to be written.
parms = a list of parameter values in args.
place = a dummy string needed for temporarily holding a place SU string.
places = a list of unique places in the cclist.
pool = a dummy string for iterating over the keywords in snap_dict.
pools = a list of snap pools in order of priority.
pool_file_name = the file name containing one of the initial pool of SU's.
pool_file_names = a list of file names containing the initial pool of SU's.
pool_name = a string converting which_pool to alphanumeric.
pool_num = an integer denoting which snap pool is being read.
prevcyc(0,1) = snap pool names for the previous cycle.
re_err = a compiled regular expression to find errors.
re_warn = a compiled regular expression to find warnings.
result = a list of dictionaries containing the results of a query.
rptfile = the report file object.
run_letter = the letter to be used in the intermediate cclist name (string).
save_name = the string name of an intermediate cclist.
save_stdout = a copy of the sys.stdout file object.
score = a dummy integer needed for temporarily holding an SPSS score.
score_x,y = the SPSS score of an SU used in cmp_score.
sn_num = the number to be used in the intermediate cclist name (integer).
snap_cutoff = a string repesenting the fraction of snap program completion.
snap_cutoff_str = a string used to help determine snap_cutoff.
snap_dict = a dictionary containing the snap information sorted by pool.
snap_dict_list = a list of dictionaries showing the schedule snap data
determined by the calendar -report jobs.
snap_dict_short = a subset of SU data from snap_dict with unique places.
snap_key = a string denoting which key in snap_dict is to be used.
snap_sus = a list of snap SU's that are schedulable on the cclist.
snap_temp_file = a list of gimme_snaps temp files
su = a dummy string needed for temporarily holding an SU string.
successful_sus = a list of snap SU's that are scheduled on ccl_final
according to its gaps report.
sunit_id = the SU name expressed as a string.
sus = a list of SU's scheduled on ccl_start.
su_dict = a dictionary containing the scheduling data for a given
snap SU determined by the calendar -report jobs.
su_dict_list = a list of dictionaries showing the schedule snap data
determined by the calendar -report jobs.
temp? (where ? = 1 or 2) = temporary variables.
temp_cclists = a list of temporary cclists.
time = a list for keeping track of clock time at start and stop of tool run.
tcycle = the cycle from which the SU list is drawn (integer).
tpool = the pool priority of the SU list (string). This has values of 'l'
or 'g'.
tend = the end time before which to schedule snaps expressed as a
time_util object.
tstart = the start time after which to schedule snaps expressed as a
time_util object.
w = a dummy string needed for iterating over a list of warnings.
warnings = a list of logfile lines containing the case-insenstive string
'warning'.
which_pool = a string denoting from which pool a snap SU is.
week = the week id parsed from the snap_temp_file expressed as a string.
week_id = week converted to a time_util object.
wrapped_cmd = currrent_command plus UNIX shell wrappers.
x = a dummy variable to hold a value temporarily.
y = a dummy variable to hold a value temporarily.
?_num (where ? = b, c, d, e, x, y, and z) = counters needed for intermediate
cclist names (integer).

**CODE ALGORITHM** --
1. Parse the command line.
2. Check that the command line start and end times intersect the
input cclist.
3. Get the current cycle from ASSIST_DB.
4. Get the *temp file produced by gimme_snaps.
5. Get the week id and snap cutoff percentage from the *temp file.
6. Determine the names of output files and open them.
7. Parse the *temp file into initial pool SU files.
8. Create the calendar -report jobs that will determine which
SU's are schedulable on the cclist.
9. Run the calendar -report jobs.
10. Parse the calendar -report output files into the dictionary
snap_dict, which gives the place and SPSS score data for each
schedulable SU.
11. Output snap_dict in a report file.
12. Create a short list of snap_dict where each snap candidate
has a unique place on the cclist.
13. While there are schedulable snaps, add the snap candidates
to the cclist and schedule them. Keep track of which snap
candidates actually schedule. Repeat steps 8-13 until there
are no more schedulable snaps.
14. Save the final cclist.
15. Report which snaps have been scheduled.
16. Delete temporary cclists.
17. Search for warnings and errors.

**NOTES** --
This tool will not necessarily raise an error if a bad SU is
encountered. Any such errors are reported at the end of the
log file. Check the output log file for any error messages. For
example, if a candidate SU fails to add to a cclist, the
error message is:
ERROR: CCL is being restored to original state before running
this command.

The work-around is to edit the file yyddd_snap_sus.temp created by
gimme_snaps in the calendar working directory. Two edits are required:
(a) delete the bad SU, and (b) determine which pool the bad SU was
in, and decrease the number of SU's in the pool by one. This number
can be found by starting from the bad SU and scrolling upwards until
one encounters a line with only one integer value at the start of a
snap pool.

Then run this tool again.

**MODIFICATION HISTORY** --
Date Who What
==== === ====
05/02/07 Bower initial implementation.
05/15/07 Bower Delete the temporary cclists.
Fix bug in get_gap_data.
05/16/07 Bower Add try/finally logic.
06/05/07 Bower Fix bug in log file appending.
06/26/07 Bower Print command line to log file.
Don't print snap dictionary to log file;
see the rpt file instead.
Fix typo in default value of output_root.
If a candidate list is empty, then skip.
Start each calendar -report job with
ccl_start, not input_cclist.
Improve elapsed time report.
07/31/07 Bower Redesign algorithm for enhanced speed.
Get current_cycle from assist.
Improve assignment of output filenames.
12/07/07 Bower Get snap cutoff percentage from temp file.
12/12/07 Bower Fix typo.
06/27/08 Bower Add documentation note on bad SU's.
08/08/08 Bower Print intermediate cclist names to output.
11/05/10 Bower Fix bug in SU sorting.
12/13/11 Bower Abort if bad SU is encountered during candlist -addsus.
12/14/11 Bower Add Merle's function check_for_usable_su.
03/22/12 Chance Remove harry and potter from FAST_CPUS
09/26/12 Bower Fix bug in determining cycle boundary.
11/16/12 Bower Compatible with new version of su_gaps.
12/04/12 Reinhart Allow the cale -report commands to execute in parallel
08/26/15 Bower Specify start and end times explicitly
"""
#***********************************************************************
__version__ = "08/26/15"
import calendar_util
import ccl_util
import file_util
import glob
import os
import re
import spss_sys_util
import spst_getopt
import stpydb
import string
import su_gaps
import su_util
import sys
import time_util
import types

LOGNAME = os.environ['LOGNAME']
FAST_CPUS = spss_sys_util.get_nodes_from_file( \
spss_sys_util.resolver("PE_DAT","fast_systems.dat"))

def run (*args):
"""Schedule SNAP SU's onto the input cclist.

Usage:
do sched_snap_sus
[-output_root=]

where is the name of the cclist to which snaps will be
added, and and are the start and end times in
YYYY.DDD:HH:MM:SS format where snaps can be scheduled.
Options:
-output_root= : redirect the output to
files with this root name. The default is to set
output_root to _snap_sus. Higher versions
default to appending '_. The suffix
names of the output files include '.log' and '.rpt'.

Example:
do sched_snap_sus gab141d1 2007.141:00:30 2007.147:23:40
-output_root=07141_snap.sus_2
"""

if not args:
print run.__doc__
return spss_sys_util.SUCCESS

# Keep track of clock time elapsed.
time = []
time.append(time_util.spss_time())

# Parse the command line.
allowed_options = ['output_root=']
options, parms = spst_getopt.spst_getopt(args,allowed_options)
if len(parms) != 3:
print "Error: Exactly 3 parameters must be passed into this module."
print
print run.__doc__
return not spss_sys_util.SUCCESS
input_cclist = ccl_util.ccl(parms[0])
if not input_cclist.exists():
raise ccl_util.NoCCLError('cclist %s does not exist' % input_cclist)
if not input_cclist.is_cataloged():
raise ccl_util.CCLError('cclist %s is not cataloged in the PMDB.' \
% input_cclist)
tstart = time_util.spss_time(parms[1])
tend = time_util.spss_time(parms[2])

# Check if tstart and tend intersect the cclist.
ccl_tstart = input_cclist.get_begin_time()
ccl_tend = input_cclist.get_end_time()
if tstart < ccl_tstart or tstart > ccl_tend:
raise ValueError ("Start time %s doesn't intersect the cclist time \
window of %s - %s" % (tstart,ccl_tstart,ccl_tend))
if tend < ccl_tstart or tend > ccl_tend:
raise ValueError ("End time %s doesn't intersect the cclist time \
window of %s - %s" % (tend,ccl_tstart,ccl_tend))

# Find the gimme_snap temp file.
snap_temp_file = spss_sys_util.glob('*_snap_sus.temp')
if not snap_temp_file:
raise IndexError ('No gimme_snap temp file was found.')
if len(snap_temp_file) > 1:
print 'More than one gimme_snap temp file was found:'
for file in snap_temp_file:
print file
print 'Delete or rename these files until the desired one remains.'
raise IndexError ('sched_snap_sus.py requires exactly one file.')
snap_temp_file = snap_temp_file[0]

# Get week_id and priority cutoff percentage from the gimme_snap temp file.
f = open(snap_temp_file,'r')
first_line = f.readline()
second_line = f.readline()
if not first_line:
raise IOError ('gimme_snap temp file %s is empty.' % snap_temp_file)
f.close()
week = first_line.strip().split()[1]
week_id = time_util.weekid(week)
if not week_id:
raise time_util.WeekIdError('Weekid %s is invalid' % week)
snap_cutoff_str = re.split(r'<|>',second_line.strip())[1]
snap_cutoff = snap_cutoff_str[:2]
cal_start = str(week_id.guess_starttime())


# Get the current cycle from assist.
dbConn = stpydb.stpydb(dbmsName=os.environ['ASSIST_DB'])
dbConn.query('select distinct cycle_number')
dbConn.query(' from cycle_boundary where')
dbConn.query(' start_boundary <= @tstart and')
dbConn.query(' end_boundary > @tstart')
result = [{}]
dbConn.setParam([['tstart',cal_start]])
dbConn.executeAll(result)
if len(result) == 1:
current_cycle = int(result[0]['cycle_number'])
elif len(result) > 1:
print "Error: multiple records found in cycle_number relation:"
print result
raise ValueError()
else:
print "Error: no records in cycle_number relation for %s" \
% ccl_tstart.get_sogs_character_time()
raise ValueError()
dbConn.close()

# Consruct snap pool names.
curcyc0 = 'current_cycle_l' + snap_cutoff + 's'
prevcyc0 = 'previous_cycle_l' + snap_cutoff + 's'
curcyc1 = 'current_cycle_g' + snap_cutoff + 's'
prevcyc1 = 'previous_cycle_g' + snap_cutoff + 's'

# Determine the names of the output files and open them.
if options.has_key('-output_root'):
output_root = options['-output_root']
else:
output_root = str(week_id) + '_snap_sus'
files = glob.glob(output_root + '*log')
if len(files) > 0:
output_root = output_root + '_' + str(len(files)+1)
try:
save_stdout = sys.stdout
sys.stdout = open(output_root + '.log','w')
rptfile = open(output_root + '.rpt','w')

# Print command line to log file.
print "Command arguments and options:\n"
print args
print
# Print the cycle number
print "Cycle %s" % current_cycle
print
save_stdout.write("Cycle %s\n" % current_cycle)

# Parse the gimme_snap temp file into the appropriate
# initial pool SU lists.
save_stdout.write("Parse the snaps into initial pool lists...\n")
print "Parse the snaps into initial pool lists..."
pool_file_names = make_su_lists(snap_temp_file, current_cycle, snap_cutoff, save_stdout, tstart, tend)

# Initial preparations to the input_cclist.
input_cclist.get()
print " "
x = spss_sys_util.command('candlist -delsus all')
if len(x[1]) > 0:
print x[1]
ccl_start = input_cclist.save('snp' + str(week_id.get_ddd()) + 'a1')
su_gaps.run(str(ccl_start))
print " "

# Iteration loop for finding and scheduling snap SU's.
no_snaps = False
niter = 0
successful_sus = []
temp_cclists = []
ccl_intermed = ccl_start
while not no_snaps:
niter = niter + 1
print "Iteration #%s" % niter
save_stdout.write("Iteration #%s\n" % niter)
rptfile.write("Iteration #%s\n" % niter)
print "Searching for snap candidates to schedule on %s..." % ccl_intermed
save_stdout.write("Searching for snap candidates to schedule on %s...\n"
% ccl_intermed)
rptfile.write("Searching for snap candidates to schedule on %s...\n"
% ccl_intermed)
comlist, output_file_dict, cclists = make_report_jobs(ccl_intermed,
current_cycle, week_id,
tstart, tend,
pool_file_names, snap_cutoff, niter)
temp_cclists.extend(cclists)
# Run the calendar -report jobs.
save_stdout.write("For each pool list, add the snap candidates to a temporary\n")
save_stdout.write("cclist and run calendar -report...\n")
print "For each pool list, add the snap candidates to a temporary"
print "cclist and run calendar report..."
run_report_jobs(comlist)

# Parse the calendar -report output files.
save_stdout.write("Sort the snap candidates by SPSS Score...\n")
print "Sort the snap candidates by SPSS Score..."
snap_dict = parse_report_output(week_id, output_file_dict, snap_cutoff)

# Print snap_dict, a dictionary with sorted snap SU's.
rptfile.write('SU\'s listed in %s sorted by priority:\n\n' % snap_temp_file)
rptfile.write('Snap pool' + 17*' ' + 'SU' + 11*' ' + 'Place' + 8*' ' + \
'Score\n\n')
write_pool_rpt(rptfile,current_cycle,snap_dict,curcyc0)
write_pool_rpt(rptfile,current_cycle,snap_dict,prevcyc0)
write_pool_rpt(rptfile,current_cycle,snap_dict,curcyc1)
write_pool_rpt(rptfile,current_cycle,snap_dict,prevcyc1)
save_stdout.write("...output is in the file %s.rpt.\n" % output_root)
print "...output is in the file %s.rpt." % output_root

# Get a short dictionary of snaps where each snap candidate has
# a unique place.
places = []
# Initialize the condensed snap dictionary.
snap_dict_short = {curcyc0:[],curcyc1:[],
prevcyc0:[],prevcyc1:[]}
pools = [curcyc0, prevcyc0, curcyc1, prevcyc1]
for pool in pools:
su_list = []
sus = []
for dict in snap_dict.get(pool):
if not dict.get('Scheduled') and \
dict.get('Place') not in places and \
dict.get('SU') not in sus:
su_list.append(dict)
sus.append(dict.get('SU'))
places.append(dict.get('Place'))
snap_dict_short.update({pool: su_list})

# Get the list of snap candidates.
snap_sus = parse_su_candidates(snap_dict_short)
if len(snap_sus) == 0:
print "No snaps can schedule."
save_stdout.write("No snaps can schedule.\n")
no_snaps = True
else:
no_snaps = False

if not no_snaps:

# Add the candidates to the ccl and schedule them.
save_stdout.write("Add the snap candidates to the cclist and ")
save_stdout.write("schedule them...\n")
print "Add the snap candidates to the cclist and schedule them..."
# Write the list of candidates to a file.
file_snap_sus = file_util.tempfile()
g = open(file_snap_sus,'w')
for su in snap_sus:
g.write('%s\n' % su)
g.close()
# Add the candidates.
ccl_intermed.get()
addsus_status = calendar_util.candlist_add(file_snap_sus)
# Abort if any candidates fail to add.
if addsus_status[0]:
save_stdout.write("%s\n" % addsus_status)
print addsus_status
raise RuntimeError ("Aborting tool since there was an error during candlist -addsus.")
# Schedule the candidates by priority in their optimum places.
snap_dict = schedule(snap_dict,snap_dict_short,curcyc0,tstart,tend)
snap_dict = schedule(snap_dict,snap_dict_short,prevcyc0,tstart,tend)
snap_dict = schedule(snap_dict,snap_dict_short,curcyc1,tstart,tend)
snap_dict = schedule(snap_dict,snap_dict_short,prevcyc1,tstart,tend)
# Delete all candidates.
x = spss_sys_util.command('candlist -delsus all')
if len(x[1]) > 0:
print x[1]
# Save the cclist.
ccl_intermed = ccl_intermed.save('snp' + str(week_id.get_ddd()) + 'a1')
temp_cclists.append(ccl_intermed)
# Print which snaps scheduled.
x, y = print_snap_gaps(ccl_intermed, snap_sus, save_stdout)
successful_sus.extend(x)
# Construct new pool_files and pool_file_names list.
pool_file_names = []
for key in snap_dict.keys():
pool_file = convert_keyword(key,current_cycle,snap_cutoff)
pool_file = pool_file + '_' + str(niter)
pool_file_names.append(pool_file)
g = open(pool_file,'w')
for dict in snap_dict.get(key):
if not dict.get('Schedule'):
g.write('%s\n' % dict.get('SU'))
g.close()
time.append(time_util.spss_time())
elapsed_time = time[len(time)-1] - time[0]
print "Elapsed time: %s\n" % elapsed_time
save_stdout.write("Elapsed time: %s\n\n" % elapsed_time)
rptfile.write("\n")

# Go back to beginning of loop

time.append(time_util.spss_time())
elapsed_time = time[len(time)-1] - time[0]
print "Elapsed time: %s\n" % elapsed_time
save_stdout.write("Elapsed time: %s\n\n" % elapsed_time)
# Save the final cclist.
ccl_final = ccl_intermed.save('snp' + str(week_id.get_ddd()) + 'af')

print "*"*15
print "Summary report:"
print "*"*15
print " "
save_stdout.write("*"*15 + "\n")
save_stdout.write("Summary report:\n")
save_stdout.write("*"*15 + "\n\n")
# Get the SU's out of the list successful_sus
sus = []
for su in successful_sus:
sus.append(su.split()[0])
# Print the gaps information.
x, gaps = print_snap_gaps(ccl_final, sus, save_stdout)
print " "
save_stdout.write("\n")
print "This final cclist has been saved to %s:%s," % \
(ccl_final.get_name().lower(),ccl_final.get_version())
print "and the gaps report is in the file %s." % gaps
save_stdout.write("This final cclist has been saved to %s,\n" \
% ccl_final)
save_stdout.write("and the gaps report is in the file %s.\n" % gaps)

# Delete the temporary cclists.
print "Delete the temporary cclists:"
save_stdout.write("Delete the temporary cclists:\n")
for cclist in temp_cclists:
if cclist.exists():
print "%s:%s" % (cclist.get_name().lower(),cclist.get_version())
save_stdout.write("%s:%s\n"
% (cclist.get_name().lower(),cclist.get_version()))
cclist.delete()

finally:
# Close the log file object.
sys.stdout.close()
sys.stdout = save_stdout

# Search for warnings and errors in the log file.
f = open(output_root + '.log','r')
lines = f.readlines()
f.close()
# reopen the log file object.
sys.stdout = open(output_root + '.log','a')
warnings = []
errors = []
re_warn = re.compile(r'.*warning.*',re.IGNORECASE)
re_err = re.compile(r'.*error.*',re.IGNORECASE)
for line in lines:
for w in re_warn.findall(line):
warnings.append(w)
for e in re_err.findall(line):
errors.append(e)
if len(warnings) > 0:
print "%s warnings occur in the file %s.log." % (len(warnings), \
output_root)
save_stdout.write("%s warnings occur in the file %s.log.\n" % \
(len(warnings), output_root))
for w in warnings:
print w
save_stdout.write("%s\n" % w)
print " "
print "Further investigation is optional.\n"
save_stdout.write("\nFurther investigation is optional.\n\n")
else:
print "No warnings were found in the file %s.log." \
% output_root
save_stdout.write("No warnings were found in the file ")
save_stdout.write("%s.log.\n" % output_root)
if len(errors) > 0:
print "%s errors occur in the file %s.log" % (len(errors), \
output_root)
save_stdout.write("%s errors occur in the file %s.log\n" \
% (len(errors),output_root))
for e in errors:
print e
save_stdout.write("%s/n" % e)
print " "
print "Please investigate these errors."
save_stdout.write("\nPlease investigate these errors.\n")
else:
print "No errors were found in the file %s.log." % output_root
save_stdout.write("No errors were found in the file %s.log.\n" \
% output_root)

# Determine the ending time.
time.append(time_util.spss_time())
elapsed_time = time[len(time)-1] - time[0]
print "Elapsed time: %s" % elapsed_time
save_stdout.write("Elapsed time: %s\n" % elapsed_time)

# Close the output file objects.
rptfile.close()
sys.stdout.close()
sys.stdout = save_stdout

return spss_sys_util.SUCCESS

# end def run

def check_for_usable_su (sunit, ccl_start, ccl_end):
"""This function performs a check that the input SU won't cause
candlist -addsus to choke.

Input:
sunit - scheduling unit as an su object
or a string in sssssss[:vv] format
ccl_start - cclist start time as an spss_time object
or a string in YYYY.JJJ:HH:MM:SS format
ccl_end - cclist end time as an spss_time object
or a string in YYYY.JJJ:HH:MM:SS format

Output:
True|False - scheduling unit is ok or not to pass to -addsus
"""

valid_flag = False

# Check if sunit is an su object
if (isinstance(sunit, su_util.su)):
su,vers = str(sunit).split(":")
elif (isinstance(sunit, types.StringType)):
# Nope, check to see if it is a string
tsu = sunit.split(":")
if (len(tsu) > 1):
su = tsu[0]
vers = tsu[1]
else:
su = tsu[0]
vers = "01"
# end if
else:
# Don't know what to do with it
raise TypeError("I don't know what to do with the input sunit %s" % sunit)
# end if

# Check to see if ccl_start is a spss_time object
if (isinstance(ccl_start, time_util.spss_time)):
start_time = ccl_start
elif (isinstance(ccl_start, types.StringType)):
start_time = time_util.spss_time(ccl_start)
else:
raise TypeError("I don't know what to do with the input ccl_start %s" % ccl_start)
# end if

# Check to see if the ccl_end is a spss_time object
if (isinstance(ccl_end, time_util.spss_time)):
end_time = ccl_end
elif (isinstance(ccl_end, types.StringType)):
end_time = time_util.spss_time(ccl_end)
else:
raise TypeError("I don't know what to do with the input ccl_end %s" % ccl_end)
# end if

ccl_time_window = time_util.window_list(time_util.window(start_time, end_time))

spss_db = spss_sys_util.get_environ_variable("SPSS_DB")[0]
dbcon = stpydb.stpydb(dbmsName=spss_db)

# First let's query qbwindows for the DO windows
dbcon.query("select qbw.window_beg, qbw.window_end")
dbcon.query(" from qbwindows qbw,")
dbcon.query(" qsbranching qsb")
dbcon.query(" where qsb.proposal_id = qbw.proposal_id")
dbcon.query(" and qsb.obset_id = qbw.obset_id")
dbcon.query(" and qsb.version_num = qbw.version_num")
dbcon.query(" and qbw.window_type = 'DO'")
dbcon.query(" and qsb.sunit_id = @SU")
dbcon.query(" and qsb.version_num = @VER")
dbcon.setParam('SU', su)
dbcon.setParam('VER', vers)
results = [[]]
dbcon.executeAll(results)

do_windows = time_util.window_list()
for wind in results:
do_windows.append(time_util.window(time_util.spss_time(wind[0]), \
time_util.spss_time(wind[1])))
# end for wind

# Check to see if any of the DO windows overlaps the calendar
if (len(do_windows.intersection(ccl_time_window)) != 0):
valid_flag = True
else:
valid_flag = False
return valid_flag
# end if

# query for valid wgacquis/wgreswnd windows
dbcon.query("select wgr.window_beg, wgr.window_end")
dbcon.query(" from wgacquis wga,")
dbcon.query(" wgreswnd wgr,")
dbcon.query(" qsbranching qsb")
dbcon.query(" where qsb.proposal_id = wga.proposal_id")
dbcon.query(" and qsb.proposal_id = wgr.proposal_id")
dbcon.query(" and qsb.obset_id = wga.obset_id")
dbcon.query(" and qsb.obset_id = wgr.obset_id")
dbcon.query(" and qsb.version_num = wga.version_num")
dbcon.query(" and qsb.version_num = wgr.version_num")
dbcon.query(" and wga.data_set_id = wgr.data_set_id")
dbcon.query(" and wgr.window_type = 'AO'")
dbcon.query(" and qsb.sunit_id = @SU")
dbcon.query(" and qsb.version_num = @VER")
dbcon.setParam('SU', su)
dbcon.setParam('VER', vers)
results = [[]]
dbcon.executeAll(results)

gs_windows = time_util.window_list()
for gswind in results:
gs_windows.append(time_util.window(time_util.spss_time(gswind[0]), \
time_util.spss_time(gswind[1])))
# end for gswind

# Check to see if any of the gs windows overlaps the calendar
if (len(gs_windows.intersection(ccl_time_window)) != 0):
valid_flag = True
else:
valid_flag = False
return valid_flag
# end if

#dbcon.close()

return valid_flag
# end def check_for_usable_su

def convert_keyword(key,current_cycle,snap_cutoff):
"""Convert a snap_dict keyword to pool_file format."""
if key.find('current') != -1:
cyc_name = str(current_cycle)
elif key.find('previous') != -1:
cyc_name = str(current_cycle - 1)
else:
cyc_name = 'unkn'
if key.find('l' + snap_cutoff + 's') != -1:
pool_name = 'l' + snap_cutoff + 's'
elif key.find('g' + snap_cutoff + 's'):
pool_name = 'g' + snap_cutoff + 's'
else:
pool_name = 'unkn'
pool_file_name = 'cyc' + cyc_name + '_' + pool_name + '.sus'
return pool_file_name

# end def convert_keyword

def convert_pool(pool,current_cycle,snap_cutoff):
"""Convert pool_file format to snap_dict keyword."""
if pool.find(str(current_cycle)) != -1:
if pool.find('l' + snap_cutoff + 's') != -1:
keyword = 'current_cycle_l' + snap_cutoff + 's'
elif pool.find('g' + snap_cutoff + 's') != -1:
keyword = 'current_cycle_g' + snap_cutoff + 's'
elif pool.find(str(current_cycle - 1)) != -1:
if pool.find('l' + snap_cutoff + 's') != -1:
keyword = 'previous_cycle_l' + snap_cutoff + 's'
elif pool.find('g' + snap_cutoff + 's') != -1:
keyword = 'previous_cycle_g' + snap_cutoff + 's'

return keyword

# end def convert_pool

def cmp_begin_time(x,y):
"""Comparison function for sorting a list of scheduling begin
time in a cclist."""
begin_time_x = time_util.spss_time(x.split()[5])
begin_time_y = time_util.spss_time(y.split()[5])
return cmp(begin_time_x,begin_time_y)

def cmp_score(x,y):
"""Comparison function for sorting a list of scheduling place
lines obtained from a calendar report."""
score_x = x['Score']
score_y = y['Score']
return cmp(score_x,score_y)

# end def cmp_score

def make_report_jobs(input_cclist, current_cycle, week_id, tstart, \
tend, cur_sus_files, snap_cutoff, niter):
"""This tool will create the cclist operations to be distributed
among different cpu's.

Input: input_cclist - name of cclist to start with
current_cycle - current cycle of proposals
week_id - YYDDD format
tstart - start time of calendar report in YYYY.DDD:HH:MM:SS
tend - end time of calendar report in YYYY.DDD:HH:MM:SS
cur_sus_files - a list of files containing SU's
niter - the number of snap scheduling iterations

Output: a list containing the calendar commands
to do the calendar -report jobs.

Returns: calendar_commands (a list of calendar commands),
a list of temporary cclists created by running calendar_commands
"""

cycle = int(current_cycle)

# Intialize the counters needed for intermediate cclist names.
b_num = 0
c_num = 0
d_num = 0
e_num = 0
x_num = 0
y_num = 0
z_num = 0

calendar_commands = []
output_file_dict = {'current_cycle_l' + snap_cutoff + 's' : [],
'current_cycle_g' + snap_cutoff + 's' : [],
'previous_cycle_l' + snap_cutoff + 's' : [],
'previous_cycle_g' + snap_cutoff + 's' : []}
temp_cclists = []

for i in cur_sus_files:
# If file i is empty then remove it from cur_sus_files
lines = open(i,'r').readlines()
if len(lines) == 0:
cur_sus_files.remove(i)

for i in cur_sus_files:
tcycle = int(string.split(i,'_')[0][-2:])
tpool = string.split(i,'_')[1][0]
if (tcycle == cycle):
if (tpool == 'l'):
run_letter = 'b'
sn_num = b_num
b_num = b_num + 1
elif (tpool == 'g'):
run_letter = 'c'
sn_num = c_num
c_num = c_num + 1
else:
run_letter = 'y'
sn_num = y_num
y_num = y_num + 1
# end if
elif (tcycle == (cycle - 1)):
if (tpool == 'l'):
run_letter = 'd'
sn_num = d_num
d_num = d_num + 1
elif (tpool == 'g'):
run_letter = 'e'
sn_num = e_num
e_num = e_num + 1
else:
run_letter = 'z'
sn_num = z_num
z_num = z_num + 1
# end if
else:
run_letter = 'x'
sn_num = x_num
x_num = x_num + 1
# end if
save_name = 'snp' + str(week_id.get_ddd()) + run_letter + str(sn_num)
if ccl_util.ccl(save_name).exists():
save_vers = ccl_util.ccl(save_name).get_newest_cclversion() + 1
else:
save_vers = ccl_util.ccl(save_name).get_newest_cclversion()
if niter:
output = save_name + '_' + str(niter)
else:
output = save_name
current_command = 'cclist -get %s;' % input_cclist
current_command = current_command + 'candlist -addsus %%%s;' % i
current_command = current_command + 'cclist -save %s;' % save_name
current_command = current_command + 'calendar -report -sched '
current_command = current_command + '-start_time=%s -end_time=%s ' \
%(tstart,tend)
current_command = current_command + '-verbose=place '
current_command = current_command + '>& %s.out;' % output
current_command = current_command + ' mv %s.report %s.report' % (save_name, output)
wrapped_cmd = spss_sys_util.shell_command_wrapper(current_command,'unix',10,reserve_child=True)
calendar_commands.append(wrapped_cmd)
keyword = convert_pool(i, current_cycle, snap_cutoff)
add_file_name = output + '.out'
output_file_dict[keyword].append(add_file_name)
temp_cclists.append(ccl_util.ccl(save_name,save_vers))

# end for i

return calendar_commands, output_file_dict, temp_cclists

# end def make_report_jobs

def make_su_lists(snap_temp_file, current_cycle, snap_cutoff, save_stdout, tstart, tend):
"""This tool will parse the gimme snap temp file into the appropriate
initial pool su lists.
"""

import string

grecs = open(snap_temp_file, 'r').readlines()

cycle = int(current_cycle)

header = grecs.pop(0)

# There are four pools to be read
num_pools = 4
cur_rec = 0
pool_file_names = []

for i in range(0,num_pools):
cur_cyc, which_pool, temp1, temp2 = string.split(grecs[cur_rec])
cur_rec = cur_rec + 1

pool_num = int(string.strip(grecs[cur_rec]))
cur_rec = cur_rec + 1

# Now determine the output file name
cyc_name = str(cycle - int(cur_cyc))
if (which_pool == '<' + snap_cutoff):
pool_name = 'l' + snap_cutoff + 's'
elif (which_pool == '>' + snap_cutoff):
pool_name = 'g' + snap_cutoff + 's'
else:
pool_name = 'unkn'
# end if
pool_file_name = 'cyc' + cyc_name + '_' + pool_name + '.sus'
pool_file_names.append(pool_file_name)
ow = open(pool_file_name, 'w')

for j in range(cur_rec,cur_rec+pool_num):
su = string.strip(grecs[j])
# Search for and filter out any candidates that cause candlist -addsus
# to rollback because of this error:
# ERROR: GS ACQ Data found in WGACQUIS for primary obset ,
# but no WGACQUIS records have usability windows that intersect
# the calendar timespan.
answer = check_for_usable_su(su, tstart, tend)
if not answer:
save_stdout.write("\nWarning: SU %s is not being added to " % su)
save_stdout.write("the cclist because\n")
save_stdout.write("doing so would cause the cclist to rollback ")
save_stdout.write("with the problem:\n")
save_stdout.write("GS ACQ Data found in WGACQUIS for primary obset ")
save_stdout.write(",\n")
save_stdout.write("but no WGACQUIS records have usability windows that ")
save_stdout.write("intersect the calendar timespan.\n")
save_stdout.write("Please report this fact to the PC.\n\n")
print "\nWarning: SU %s is not being added to the cclist.\n" % su
print "Please report this fact to the PC."
print "See the log file for details.\n"
else:
ow.write('%s\n' % su)
# end for j
cur_rec = cur_rec + pool_num
ow.close()

# end for i

return pool_file_names

# end def make_su_lists

def parse_report_output(week_id, output_file_dict, snap_cutoff):
"""Parse the calendar -report output files to produce
the intermediate files needed to run cale -addcand.
"""

# Initialize the dictionary for parsing results.
snap_dict = {'current_cycle_l' + snap_cutoff + 's':'',
'current_cycle_g' + snap_cutoff + 's':'',
'previous_cycle_l' + snap_cutoff + 's':'',
'previous_cycle_g' + snap_cutoff + 's':''}

# Search the output files, constructing a list of schedulable SU properties
# associated with each of the four cycle pools.
for cycle_pool in output_file_dict.keys():
snap_dict_list = []
for file in output_file_dict[cycle_pool]:
su_dict_list = parse_the_reports(file)
snap_dict_list.extend(su_dict_list)
snap_dict[cycle_pool] = snap_dict_list

return snap_dict

# end def parse_report_output

def parse_su_candidates(snap_dict):
"""Parse the SU's out of snap_dict to get a list of SU's."""
snap_sus = []
for key in snap_dict.keys():
for dict in snap_dict.get(key):
snap_sus.append(dict['SU'])

return snap_sus

# end def parse_su_candidates

def parse_the_reports(file):
"""Parse a calendar -report output file."""
# Read the calendar -report.
lines = open(file,'r').readlines()

# Find all SU's that are schedulable.
su_dict_list = []
for line in lines:
if line.find('Score') != -1:
# Determine the SU, place, and score
su = line.strip()[7:14].strip()
place = line.strip()[21:28].strip()
score = int(line.strip()[40:].strip())
# Assemble these into a dictionary and append to a list.
su_dict = {'SU' : su, 'Place' : place, 'Score' : score,
'Scheduled' : False}
su_dict_list.append(su_dict)

# Sort su_dict_list based on 'Score'.
su_dict_list.sort(cmp_score, reverse=True)

return su_dict_list

# end def parse_the_reports

def print_snap_gaps(ccl_final, snap_sus, save_stdout):
"""Print a summary of SU gaps data from snap_sus."""

su_gaps.run(str(ccl_final))
print " "
gaps = ccl_final.get_name().lower() + '.gaps'
temp_sched_sus = ccl_final.get_scheduled_sus()
successful_sus = []
for su in snap_sus:
x,y = spss_sys_util.command('grep %s %s' % (su,gaps))
if y:
successful_sus.append(y)

if len(successful_sus) > 0:
# Remove any duplicates in successful_sus.
x = []
for su in successful_sus:
if su not in x:
temp_su_line = su.split()
temp_su_line[5] = str(temp_sched_sus.get_sunit(temp_su_line[0])['beg_tim'])
temp_su_line[7] = str(temp_sched_sus.get_sunit(temp_su_line[0])['end_tim'])
x.append(string.join(temp_su_line))
# Sort successful_sus by begin time."
x.sort(cmp_begin_time)
# Remove year from time.
successful_sus = []
for line in x:
temp_su_line = line.split()
temp_su_line[5] = temp_su_line[5][5:]
temp_su_line[7] = temp_su_line[7][5:]
newline = string.join(temp_su_line)
successful_sus.append(newline)
print "Snap candidates that have been scheduled:\n"
save_stdout.write("Snap candidates that have been scheduled:\n\n")
for line in successful_sus:
print line.strip()
save_stdout.write("%s\n" % line.strip())

return successful_sus, gaps

# end def print_snap_gaps

def run_report_jobs(comlist):
"""This tool will run the cclist operations distributed
among different cpu's.

Input: comlist = a list of cclist and calendar commands

Output: spss_sys_util.SUCCESS
"""
print comlist
outfile = sys.stdout
errors_list = []

# Remove a given CPU from the list.
nodes = spss_sys_util.get_good_nodes(FAST_CPUS)
print "Run calendar -report jobs on the following nodes:"
for node in nodes:
print node[0]
print " "
if not nodes:
raise ValueError('No nodes found!')
spss_sys_util.parallel_rsh(nodes,
comlist,
outfile,
errors_list=errors_list)
if errors_list:
print "ERROR: The following commands had problems during execution."
for error in errors_list:
print error

return spss_sys_util.SUCCESS

# end def run_report_jobs

def schedule(snap_dict,snap_dict_short,snap_key,tstart,tend):
"""Using snap_dict, schedule the candidates."""
sus_scheduled = []
for dict in snap_dict_short.get(snap_key):
y = calendar_util.calendar_addcand(dict.get('SU'),dict.get('Place'),
start_t=tstart,end_t=tend)
if len(y[0]) == 1 and isinstance(y[0][0],types.DictType):
if y[0][0].has_key('added_window'):
sus_scheduled.append(dict.get('SU'))

# Update snap_dict.
for dict in snap_dict.get(snap_key):
if dict.get('SU') in sus_scheduled:
dict.update({'Schedule': True})

return snap_dict

# end def schedule

def write_pool_rpt(rptfile,current_cycle,snap_dict,snap_key):
"""Write the snap pool report to open file object rptfile."""
# Determine which pool we have and construct its output name.
if snap_key.find('current') > -1:
cycle = str(current_cycle)
elif snap_key.find('previous') > -1:
cycle = str(current_cycle - 1)
else:
cycle = None
if snap_key.find('_l') != -1:
inequality = '<'
elif snap_key.find('_g') != -1:
inequality = '>'
else:
inequality = None
factor = re.findall(r'\d+',snap_key)[0]
pool = 'Cycle ' + cycle + ' ' + inequality + ' ' + factor \
+ '%' + ' scheduled'
rptfile.write('%s\n' % pool)
# Print the SU data in this pool.
if len(snap_dict.get(snap_key)) > 0:
for dict in snap_dict.get(snap_key):
rptfile.write('\t\t\t %s %s %s \n' % \
(dict['SU'],dict['Place'],dict['Score']))
else:
rptfile.write('No SU\'s in this snap pool\n')


# end def write_pool_rpt


if __name__ == '__main__':
run(*sys.argv[1:])