From 9248397920cdb289a9aa15f124011415d636d6a7 Mon Sep 17 00:00:00 2001 From: Tom Glanzman Date: Tue, 15 Nov 2016 14:47:15 -0800 Subject: [PATCH] Re-wrangle phoSim inputs and outputs to make greater use of local scratch and thereby reduce load on a strained Lustre file system --- workflows/TW-phoSim-r3/config.py | 2 +- workflows/TW-phoSim-r3/phoSimPrep.py | 43 ++++++++-- workflows/TW-phoSim-r3/runPhoSim.py | 9 ++- workflows/TW-phoSim-r3/scratch.py | 93 ++++++++++++++++++++++ workflows/TW-phoSim-r3/setupPhoSimInput.py | 53 ++++++------ workflows/TW-phoSim-r3/setupVisit.py | 2 +- 6 files changed, 161 insertions(+), 41 deletions(-) create mode 100644 workflows/TW-phoSim-r3/scratch.py diff --git a/workflows/TW-phoSim-r3/config.py b/workflows/TW-phoSim-r3/config.py index 1153b53..6839f96 100755 --- a/workflows/TW-phoSim-r3/config.py +++ b/workflows/TW-phoSim-r3/config.py @@ -57,7 +57,7 @@ ## phoSim (persistent) scratch space directory path ## SLAC: /lustre/ki/pfs/fermi_scratch/lsst//// -PHOSIMSCRATCH = os.path.join('/lustre/ki/pfs/fermi_scratch/lsst',os.environ['PIPELINE_TASKPATH'].replace('.','/'),os.environ['PIPELINE_STREAMPATH'].replace('.','/')) +PHOSIMPSCRATCH = os.path.join('/lustre/ki/pfs/fermi_scratch/lsst',os.environ['PIPELINE_TASKPATH'].replace('.','/'),os.environ['PIPELINE_STREAMPATH'].replace('.','/')) PHOSIMCLEANUP = True diff --git a/workflows/TW-phoSim-r3/phoSimPrep.py b/workflows/TW-phoSim-r3/phoSimPrep.py index 3173a6c..29f1031 100755 --- a/workflows/TW-phoSim-r3/phoSimPrep.py +++ b/workflows/TW-phoSim-r3/phoSimPrep.py @@ -4,26 +4,35 @@ ## import os,sys,shutil +import tarfile ## Insert task config area for python modules (insert as 2nd element in sys.path) sys.path.insert(1,os.getenv('TW_CONFIGDIR')) from config import * +import scratch print '\n\nWelcome to phoSimPrep.py\n========================\n' ## Generate instanceCatalog on-the-fly -## Create scratch directory in (persistent) /lustre, if necessary. +## Create persistent scratch directory in /lustre, if necessary. ## File path = PHOSIMPSCRATCH (defined in config.py) - if not os.path.exists(PHOSIMPSCRATCH): os.makedirs(PHOSIMPSCRATCH) + +## Create true scratch directory in /scratch +scr = scratch.scratch() +SCRATCH = scr.getScratch() +scr.statScratch() + + ## generate instance catalog and SED files for phoSim # generatePhosimInput.py obsHistID [options] destIC = os.path.join(PHOSIMPSCRATCH,'instanceCatalog.txt') -destSEDdir = PHOSIMPSCRATCH +#destSEDdir = PHOSIMPSCRATCH +destSEDdir = SCRATCH obsHistID = os.getenv('TW_OBSHISTID') cacheDir = TW_CACHEDIR opssimDir = TW_OPSSIMDIR @@ -57,9 +66,28 @@ print 'Awkward return code, redefining rc = ',rc pass + +## tar up the sprinkled SED files +print 'tar up the sprinkled SED files and store in /lustre' +workingdir = os.getcwd() +os.chdir(destSEDdir) ## Move to directory containing SED dir to feed tar relative paths + +seddir = 'spectra_files' ## This is where the sprinkled SEDs are generated +tarinput = seddir +tarname = seddir+'.tar.gz' +taroutput = os.path.join(PHOSIMPSCRATCH,tarname) ## output goes into /lustre + +tarobj = tarfile.open(name=taroutput,mode='w:gz') +tarobj.add(tarinput,recursive=True) ## add entire directory tree of SEDs to tar archive +tarobj.close() + +os.chdir(workingdir) + + + ## Protect scratch directory: rwxr-sr-t cmd = 'chmod -R 3755 '+PHOSIMPSCRATCH -print 'Protect scratch directory\n',cmd +print 'Protect scratch directory and its contents\n',cmd rc2 = os.system(cmd) if rc2 != 0: @@ -70,10 +98,15 @@ ## Confirm working directory contents -cmd = 'ls -l '+destSEDdir +cmd = 'ls -l '+PHOSIMPSCRATCH print cmd os.system(cmd) + +## Clean up the local scratch space +scr.cleanScratch() +scr.statScratch() + ## Run a trial phoSim to ensure all inputs+code respond reasonably? ## (not yet, if ever) diff --git a/workflows/TW-phoSim-r3/runPhoSim.py b/workflows/TW-phoSim-r3/runPhoSim.py index f0a164d..fbc46a8 100755 --- a/workflows/TW-phoSim-r3/runPhoSim.py +++ b/workflows/TW-phoSim-r3/runPhoSim.py @@ -68,11 +68,12 @@ #prep.inputRoot = PHOSIMIN prep.inputRoot = PHOSIMPSCRATCH prep.phosimInstDir = PHOSIMINST -prep.SEDlib = PHOSIMSEDS -prep.scratch = PHOSIMPSCRATCH -prep.refCF = PHOSIMCF ## cmd file template may require editing +prep.SEDlib = PHOSIMSEDS ## production SEDs +prep.sedFile = 'spectra_files.tar.gz' ## sprinkled SEDs + +prep.refCF = PHOSIMCF ## cmd file template (may require editing) prep.persistentScratch = True ## dynamically generated instance catalog + SEDs -prep.cleanupFlag = False ## DEBUG - keep contents of scratch (/lustre) +prep.cleanupFlag = False ## DEBUG - keep contents of scratch (work1,ic,seds,cFile) = prep.run() diff --git a/workflows/TW-phoSim-r3/scratch.py b/workflows/TW-phoSim-r3/scratch.py new file mode 100644 index 0000000..717a0eb --- /dev/null +++ b/workflows/TW-phoSim-r3/scratch.py @@ -0,0 +1,93 @@ +#!/nfs/farm/g/lsst/u1/software/redhat6-x86_64-64bit-gcc44/anaconda/2.3.0/bin/python + +## scratch.py - Manage local /scratch space + + +import os,sys,shutil,socket + +class scratch(object): + def __init__(self): + self.scratch = None + self.prefix = 'LSSTSIM' + self.scratches = ['/scratch'] + self.scrMode = 0o3755 + self.cleanupFlag = True + self.host = socket.gethostname() + self.trace = False + return + + def getScratch(self): + ## Set up scratch space + if self.trace: print 'Entering getScratch().' + if self.scratch == None: + scratchRoot = '' + for scratch in self.scratches: + if os.path.isdir(scratch): + scratchRoot = scratch + break + pass + + if scratchRoot == '': + print "Unable to find suitable scratch space." + print self.scratches + sys.exit(1) + + ## Define scratch directory + ## Naming convention for scratch directories: + ## Standalone job: /scratch/PID- + ## Ordinary batch job: /scratch/LSF- + ## Pipeline batch job: /scratch/LSSTSIM//// + ## Naming convention for scratch directories: + jobid = os.path.join(self.prefix,'PID-'+str(os.getpid())) + if os.environ.get('LSB_JOBID') != None: + jobid=os.path.join(self.prefix,'LSF-'+os.environ['LSB_JOBID']) + if os.environ.get('PIPELINE_TASKPATH') != None: + jobid = os.path.join(self.prefix,os.environ.get('PIPELINE_TASKPATH').replace('.','/'),os.environ.get('PIPELINE_STREAMPATH').split('.')[0],os.environ.get('LSB_JOBID')) + pass + pass + + self.scratch = os.path.join(scratchRoot,str(jobid)) + print 'defined self.scratch = ',self.scratch + + ## Create scratch directory + if os.path.exists(self.scratch): + print 'WARNING: scratch directory already exists. Removing...' + shutil.rmtree(self.scratch) + pass + os.makedirs(self.scratch,self.scrMode) + pass + else: + print 'WARNING: scratch already defined - ',self.scratch + pass + + return self.scratch + + + def cleanScratch(self): + ## Cleanup the 'scratch' space + if self.trace: print 'Entering cleanScratch().' + + if self.scratch != None: + ## Now obliterate the scratch space + if self.cleanupFlag: + print "Cleaning up scratch area" + shutil.rmtree(self.scratch) + self.scratch = None + else: + print "Retaining scratch area" + pass + pass + return + + def statScratch(self): + print '\n\n=====================================================' + print 'Status of scratch space' + print ' scratch = ',self.scratch + print ' prefix = ',self.prefix + print ' scratches = ',self.scratches + print ' scrMode = ',oct(self.scrMode) + print ' host = ',self.host + print ' cleanupFlag = ',self.cleanupFlag + print ' trace = ',self.trace + print '=======================================================\n\n' + return diff --git a/workflows/TW-phoSim-r3/setupPhoSimInput.py b/workflows/TW-phoSim-r3/setupPhoSimInput.py index 1838a0c..70ff5ce 100755 --- a/workflows/TW-phoSim-r3/setupPhoSimInput.py +++ b/workflows/TW-phoSim-r3/setupPhoSimInput.py @@ -12,9 +12,10 @@ import os,sys,shutil import argparse +import scratch trace = True -class setupPhoSimInput: +class setupPhoSimInput(object): def __init__(self,icFile=None): self.debug = True self.projectName = 'LSSTSIM' @@ -29,9 +30,11 @@ def __init__(self,icFile=None): self.specialSEDdir = 'spectra_files' ## name of directory to contain custom SEDs self.locSpecialSEDs = None ## full path to (unpacked) custom SED files + self.scratchObj = scratch.scratch() ## Create a scratch object self.scratch = None ## high-performance I/O scratch space (if possible) -# self.scratches = ['/lustre/ki/pfs/fermi_scratch','/scratch','/tmp'] # list of scratch candidates - self.scratches = ['/lustre/ki/pfs/fermi_scratch'] # for TW-run3 insist upon semi-persistent only + self.scratches = ['/scratch'] ## list of scratch candidates +# self.scratches = ['/lustre/ki/pfs/fermi_scratch','/scratch','/tmp'] + self.persistentScratch = False ## Will the scratch area persist after this task step? self.cleanupFlag = True ## True => erase scratch directory at end of task @@ -46,41 +49,24 @@ def __init__(self,icFile=None): self.nextCP = None ## which checkpoint will be performed self.reqCP = None self.archiveWorkDir = None ## location of checkpoint files - return - def getScratch(self): - ## Set up scratch space - if trace: print 'Entering getScratch().' - if self.scratch == None: - scratchRoot = '' - for scratch in self.scratches: - if os.path.isdir(scratch): - scratchRoot = scratch - break - pass - - if scratchRoot == '': - print "Unable to find suitable scratch space." - print self.scratches - sys.exit(1) - ## Create scratch directory - jobid = 'PID-'+str(os.getpid()) - if os.environ.get('LSB_JOBID') != None: jobid='LSF-'+os.environ['LSB_JOBID'] - self.scratch = os.path.join(scratchRoot,self.projectName,str(jobid)) - print 'defined self.scratch = ',self.scratch - os.makedirs(self.scratch) - pass - ## Create 'work' directory in scratch area + def getScratch(self): + ## Set up scratch space + self.scratch = self.scratchObj.getScratch() + self.scratchObj.statScratch() + + ## Create phoSim 'work' directory in scratch area self.locWork = os.path.join(self.scratch,'work') - if not os.access(self.locWork,os.W_OK):os.makedirs(self.locWork) + if not os.access(self.locWork,os.W_OK):os.makedirs(self.locWork,0o3755) return + def protectScratch(self): - ## Protect scratch directory: rwxr-sr-t + ## Protect scratch directory and everything in it: rwxr-sr-t if trace: print 'Entering protectScratch().' cmd = 'chmod -R 3755 '+self.scratch print 'Protect scratch directory\n',cmd @@ -203,6 +189,12 @@ def makeSEDtree(self): rc = os.system(cmd) if rc != 0: sys.exit(1) + ## Change SEDs directory permissions + cmd = 'chmod -R 3755 '+self.locSEDs + print 'cmd = ',cmd + rc = os.system(cmd) + if rc != 0: sys.exit(1) + ## Add link to special SEDs for just this run if self.locSpecialSEDs != None and not os.path.islink(self.locSpecialSEDs): print 'Create sym link to custom SED file directory' @@ -296,7 +288,8 @@ def clean(self): ## Now obliterate the scratch space if self.cleanupFlag: print "Cleaning up scratch area" - shutil.rmtree(self.scratch) + self.scratchObj.cleanScratch() + #shutil.rmtree(self.scratch) else: print "Retaining scratch area" pass diff --git a/workflows/TW-phoSim-r3/setupVisit.py b/workflows/TW-phoSim-r3/setupVisit.py index b1fce79..e21d844 100755 --- a/workflows/TW-phoSim-r3/setupVisit.py +++ b/workflows/TW-phoSim-r3/setupVisit.py @@ -200,7 +200,7 @@ ## Create working directory structure for this invocation of phoSim ### Check if directory already exists from previous execution and, if not, create it. -## Directory structure: $TW_ROOT/output//{work,output} +## Directory structure: $TW_ROOT/phosim_output//{work,output} outDir = os.path.join(os.environ['TW_ROOT'],phoSimOutputRoot,os.environ['TW_SIXDIGSTREAM']) log.info('PhoSim output directory = \n\t%s',outDir)