#!/bin/bash "exec" "python" "$0" "$@" import re import os import sys import time import urllib import commands #import atexit # look for cloud and callback, and remove them from args cloud = '' cloudArg = False callBackURL = '' callBackArg = False newArg = '' defPATH = '/usr/local/bin:/bin:/usr/bin' for arg in sys.argv[1:]: # look for callback if (not callBackArg) and arg == '--callBack': callBackArg = True continue # set callback URL if callBackArg: callBackURL = urllib.unquote(arg) callBackArg = False print "> Callback URL" print callBackURL continue # look for callback if (not cloudArg) and arg == '--cloud': cloudArg = True continue # set cloud if cloudArg: cloud = arg cloudArg = False print "> Cloud" print cloud continue # normal parameters newArg += " %s" % arg status = 100 if cloud == "dummy": #'CA': # look for guids params = newArg.split() try: strGUID = params[1+params.index('--guids')] # change ',' to '+' strGUID = strGUID.replace(',','+') # construct command # https proxy com = "export https_proxy=http://squid.sec.bnl.local:3128;" com += 'curl -k --silent https://vobox.triumf.ca/python/test.py/staged?guids=%s' % strGUID # polling nLoop = 6 for iLoop in range(nLoop): print commands.getoutput('date') print com # execute status,out = commands.getstatusoutput(com) print status print out if status != 0: status = status % 255 print "Invalid StatusCode %s" % status break if out == 'True': break # set error code status = 100 if iLoop+1 != nLoop: time.sleep(60*60) except: type, value, traceBack = sys.exc_info() print type,value elif cloud in ['CA','FR','IT','NL','TW','UK','DE','ES']: com = "source $VO_ATLAS_SW_DIR/dq2/dq2.sh;" # keep original X509_USER_PROXY/X509_CERT_DIR if os.environ.has_key('X509_USER_PROXY'): com += "export X509_USER_PROXY=%s;" % os.environ['X509_USER_PROXY'] if os.environ.has_key('X509_CERT_DIR'): com += "export X509_CERT_DIR=%s;" % os.environ['X509_CERT_DIR'] if os.environ.has_key('X509_VOMS_DIR'): com += "export X509_VOMS_DIR=%s;" % os.environ['X509_VOMS_DIR'] com += "env;" com += "voms-proxy-info -all;" # look for guids/lfns/dsname params = newArg.split() guids = params[1+params.index('--guids')] lfns = params[1+params.index('--lfns')] dsname = params[-1] # dq2_prestage if cloud == 'CA': com += "dq2_prestage --lfns %s --guids %s -t 30 %s TRIUMF-LCG2_DATATAPE" % (lfns,guids,dsname) elif cloud == 'FR': com += "dq2_prestage --lfns %s --guids %s -t 30 %s IN2P3-CC_DATATAPE" % (lfns,guids,dsname) elif cloud == 'NL': com += "dq2_prestage --lfns %s --guids %s -t 30 %s SARA-MATRIX_DATATAPE" % (lfns,guids,dsname) elif cloud == 'DE': com += "dq2_prestage --lfns %s --guids %s -t 30 %s FZK-LCG2_DATATAPE" % (lfns,guids,dsname) elif cloud == 'TW': com += "dq2_prestage --lfns %s --guids %s -t 30 %s TAIWAN-LCG2_DATATAPE" % (lfns,guids,dsname) elif cloud == 'ES': com += "dq2_prestage --lfns %s --guids %s -t 30 %s PIC_DATATAPE" % (lfns,guids,dsname) elif cloud == 'UK': com += "dq2_prestage --lfns %s --guids %s -t 30 %s RAL-LCG2_DATATAPE" % (lfns,guids,dsname) else: com += "dq2_prestage --lfns %s --guids %s -t 30 %s INFN-T1_DATATAPE" % (lfns,guids,dsname) print "> Command" print com status,out = commands.getstatusoutput(com) if status != 0: # not cached status = 160 # log analysis for line in out.split('\n'): match = re.search('No files found for',line) if match != None: # not found status = 150 break if status != 0: status += 10000 print "> Status" print status print "> Out" print out elif cloud in ['dummy']: com = "unset LD_LIBRARY_PATH;export PATH=%s;" % defPATH com += "source /afs/usatlas/osg/client/@sys/current/setup.sh;" com += "source /afs/usatlas.bnl.gov/i386_redhat72/opt/dcache/dcache_client_config.sh;" com += "source /afs/usatlas/Grid/Don-Quijote/DQ2_0_3_client/dq2.sh;" # keep original X509_USER_PROXY/X509_CERT_DIR if os.environ.has_key('X509_USER_PROXY'): com += "export X509_USER_PROXY=%s;" % os.environ['X509_USER_PROXY'] if os.environ.has_key('X509_CERT_DIR'): com += "export X509_CERT_DIR=%s;" % os.environ['X509_CERT_DIR'] if os.environ.has_key('X509_VOMS_DIR'): com += "export X509_VOMS_DIR=%s;" % os.environ['X509_VOMS_DIR'] srmV2dir = "/usatlas/u/maeno/prestage/srmv2" com += "export LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH;" % srmV2dir com += "export PYTHONPATH=%s:%s/lib/python:$PYTHONPATH;" % (srmV2dir,srmV2dir) com += "export PATH=%s:$PATH;env;" % srmV2dir com += "voms-proxy-info -all;" # look for guids/lfns/dsname params = newArg.split() guids = params[1+params.index('--guids')] lfns = params[1+params.index('--lfns')] dsname = params[-1] # dq2_prestage if cloud == 'CA': com += "dq2_prestage --lfns %s --guids %s -t 1800 %s TRIUMF-LCG2_DATATAPE" % (lfns,guids,dsname) elif cloud == 'FR': com += "dq2_prestage --lfns %s --guids %s -t 1800 %s IN2P3-CC_DATATAPE" % (lfns,guids,dsname) else: com += "dq2_prestage --lfns %s --guids %s -t 1800 %s INFN-T1_DATATAPE" % (lfns,guids,dsname) print "> Command" print com status,out = commands.getstatusoutput(com) if status != 0: # not cached status = 160 # log analysis for line in out.split('\n'): match = re.search('No files found for',line) if match != None: # not found status = 150 break if status != 0: status += 10000 print "> Status" print status print "> Out" print out else: com = "unset LD_LIBRARY_PATH;export PATH=%s;" % defPATH com += "source /afs/usatlas/osg/client/@sys/current/setup.sh;" com += "source /afs/usatlas.bnl.gov/i386_redhat72/opt/dcache/dcache_client_config.sh;" com += "source /afs/usatlas.bnl.gov/Grid/Don-Quijote/dq2_user_client/setup.sh.BNL;" # keep original X509_USER_PROXY/X509_CERT_DIR if os.environ.has_key('X509_USER_PROXY'): com += "export X509_USER_PROXY=%s;" % os.environ['X509_USER_PROXY'] if os.environ.has_key('X509_CERT_DIR'): com += "export X509_CERT_DIR=%s;" % os.environ['X509_CERT_DIR'] if os.environ.has_key('X509_VOMS_DIR'): com += "export X509_VOMS_DIR=%s;" % os.environ['X509_VOMS_DIR'] # https proxy com += "export https_proxy=http://squid.sec.bnl.local:3128;" com += "export TZ=UTC;" com += "env;" com += "voms-proxy-info -all;" com += "dq2_cr --noSleep -v --useGSC" com += newArg tmpOutput = 'tmp.stdout.%s' % commands.getoutput('uuidgen') com += ' > %s 2>&1' % tmpOutput print "> Command" print com status,out = commands.getstatusoutput(com) status = status % 255 # log analysis newStatus = False try: tmpOutFile = open(tmpOutput) for line in tmpOutFile: out += line if not newStatus: line = line[:-1] match = re.search('^!!([^!]+)!!(\d+)!!',line) if match != None: print "===== %s %s" % (match.group(1),match.group(2)) status = int(match.group(2)) newStatus = True tmpOutFile.close() commands.getstatusoutput('rm -f %s' % tmpOutput) except: pass if status != 0: status += 10000 print "> Status" print status print "> Out" print out # send callback if status == 0 and callBackURL != '': callBackURL = re.sub('&','\&',callBackURL) print callBackURL print commands.getoutput('curl -k %s' % callBackURL) sys.exit(status)