Документ взят из кэша поисковой машины. Адрес оригинального документа : http://www.stsci.edu/spst/UnixTransition/doc/mon_eff_files.py Дата изменения: Fri Apr 8 12:46:12 2016 Дата индексирования: Mon Apr 11 01:34:10 2016 Кодировка: Поисковые слова: http astrokuban.info astrokuban

#!/usr/bin/env python
#
#MODULE mon_eff_files
#
#***********************************************************************
"""

**PURPOSE** --
Check for the existence of newly created efficiency file products and
distribute them

**DEVELOPER** --
Alan Patterson

**MODIFICATION HISTORY** --

Initial implementation app 04/22/03
Mods from Python review app 05/12/03
Corrected file check app 05/30/03
Mods to accommodate stretched COM variable app 10/23/03
Mods for new met_util app 11/07/03
Update to current python2.2.3 executable app 01/22/04
Updated for python 2.4.1 drc 08/17/05
Modified for SGE 6 drc 05/08/07
Fixed typo in import from batch_util gab 05/16/07
Fix embedded she-bang line in output script mdr 10/14/15
"""
#**********************************************************************
__author__ = 'Alan Patterson'
__version__ = "15.10.14"

import os
import time
import sys
import string
import spss_sys_util
import file_util
import time_util
import distrib_effdata
import spst_getopt
import met_util
from batch_util import SPSS_QUEUE

LOOP_TIME = 120.0
CHECK_TIME = 7200.0 # real value should be >= 1000
MET_DIR = spss_sys_util.get_environ_variable('METRIC_DIR')[0]

def run(*args):
"""Checks for the existence of the three standard metrics for efficiency
and distributes them if their creation time is after the start of this
tool

Usage:
do mon_eff_files [-testfile=] [-mail=] [-submit]
where -testfile= checks for file
in the $METRICS_DIR/tmp dir
rather than the efficiency files
-mail= causes informational mail
messages to be sent to
at completion
-submit causes this tool to be submitted
to an spst Unix batch queue
"""
allowed_options = ["testfile=","mail=","submit"]
low_args = []
for a in args:
low_args.append(string.lower(a))
options, parms = spst_getopt.spst_getopt(low_args, \
allowed_options)

# parse options
if options.has_key("-mail"):
mailto = options["-mail"]
else:
mailto = None
if options.has_key("-submit"):
if options.has_key("-testfile"):
submit(options["-testfile"],mailto)
else:
submit(None,mailto)
return spss_sys_util.SUCCESS
if options.has_key("-testfile"):
filelist = [options["-testfile"]]
else:
# create internal dictionary of efficiency files from master list
# in met_util
fdict = {}
for fil in met_util.EFFICIENCY_LIST:
fdict[fil[0]] = 0

start_time = time.time()
end_check_time = start_time + CHECK_TIME
print '%s Started loop checking for efficiency files' % \
time_util.spss_time()

# Loop until end time is reached or all files in filelist exist
# and are newly created.
while (time.time() < end_check_time and \
len(fdict) != found_count(fdict)):
time.sleep(LOOP_TIME)
for fil in fdict.keys():
x = os.path.join(MET_DIR,'tmp',fil)
if os.path.exists(x):
st = os.stat(x)
# ensure file was created after start time of this test loop
if st[9] > start_time:
fdict[fil] = 1

# check if all files meet test criteria at ending of loop
test_count = 0
for fil in fdict.keys():
x = os.path.join(MET_DIR,'tmp',fil)
if os.path.exists(x):
st = os.stat(x)
# ensure file was created after start time of this test loop
if st[9] > start_time:
test_count = test_count + 1
# test_count should equal number of file dictionary entries
if len(fdict) != test_count:
print '%s Files not found - exiting' % time_util.spss_time()
if mailto is not None:
tempfile = file_util.tempfile()
f = open(tempfile,'w')
t = str(time_util.spss_time())
f.write('%s\n' % t)
f.write('No recent efficiency metric files have been generated\n')
f.write('Check the batch job system for recently created\n')
f.write('efficiency files in $METRIC_DIR/tmp\n\n')
f.write('ls $METRIC_DIR/tmp/eff*\n\n')
f.write('There should be 3 files (ps,txt,xls)\n\n')
f.write('Possible missing files are:\n\n')
for fil in fdict.keys():
if fdict[fil] == 0:
f.write('%s\n' % fil)
f.write('\nDistribute them once they have been created\n')
f.write('by executing\n\ndo distrib_effdata\n')
f.close()
spss_sys_util.mail(mailto, \
'Check generation of eff file', \
tempfile,dis=0)
os.remove(tempfile)
else:
print '%s Files found and distributed' % time_util.spss_time()
if not options.has_key('-testfile'):
distrib_effdata.run()
if mailto is not None:
tempfile = file_util.tempfile()
f = open(tempfile,'w')
t = str(time_util.spss_time())
f.write('%s\n' % t)
f.write('Efficiency files distributed\n')
f.close()
spss_sys_util.mail(mailto, \
'Efficiency files distributed', \
tempfile,dis=0)
os.remove(tempfile)

return spss_sys_util.SUCCESS

def found_count(fdict):
"""Counts the integer values in the dictionary
"""
n = 0
for k in fdict.keys():
n = n + fdict[k]
return n

# end found_count

def submit(testfile=None,mailto=None):
"""Submit the mon_eff_files job to a Unix queue
"""
# setup runs specific qualifiers
end_com = ''
if testfile is not None:
end_com = end_com + ' -testfile=%s' % testfile
if mailto is not None:
end_com = end_com + ' -mail=%s' % mailto

# define standard Unix job submission command
q_command = 'qsub -j y -cwd -m n'
q_command = q_command + ' -N mon_eff_files'
# define log file
logfile = '$METRIC_DIR/eff.log'
q_command = q_command + ' -o ' + logfile + ' -q %s ' % SPSS_QUEUE

# The next line was used for execution tests under development
#q_command = q_command + '$COM/exec_csh.csh $METRIC_DIR/mon_eff_files.py'
# command for execution of configured command
#q_command = q_command + '$COM/exec_csh.csh $COM/mon_eff_files.py'
q_command = q_command + '${exec_csh} ${mon_eff}'

# ensure command carries uppercase through to Unix for environment
# variable substitution
a_command = '%s%s' % (q_command,end_com)

com_file = make_unix_comfile(a_command)
exec_command = 'source %s' % com_file
unixnode = spss_sys_util.get_good_nodes()[0][0]
status, out = spss_sys_util.rsh('planinst',unixnode,exec_command)
#print status
print out

return

def make_unix_comfile(a_com):
"""
"""
res = 'mon_eff_submission.unix'
fcomfile = os.path.join(MET_DIR,res)
out = fcomfile
fcom = open(fcomfile,'w')
fcom.write('#!/usr/bin/tcsh\n')
fcom.write('set scriptpath = "${COM}"\n')
fcom.write('set fnd = "F"\n')
fcom.write('foreach j ($scriptpath)\n')
fcom.write(' set exec_csh = "${j}/exec_csh.csh"\n')
fcom.write(' if (-e ${exec_csh}) then\n')
fcom.write(' set fnd = "T"\n')
fcom.write(' break\n')
fcom.write(' endif\n')
fcom.write('end\n')
fcom.write('unset j\n')
fcom.write('set fnd2 = "F"\n')
fcom.write('foreach j ($scriptpath)\n')
fcom.write(' set mon_eff = "${j}/mon_eff_files.py"\n')
fcom.write(' if (-e ${mon_eff}) then\n')
fcom.write(' set fnd2 = "T"\n')
fcom.write(' break\n')
fcom.write(' endif\n')
fcom.write('end\n')
fcom.write('unset j\n')
fcom.write('%s\n' % a_com)
fcom.close()
return out

if __name__ == '__main__':
if len(sys.argv) > 1:
apply(run, tuple(sys.argv[1:]))
else:
run()