# ProdSubmitter.py # # David Adams # April 2009 # # Class that desscribes a production sample, i.e. all the tasks # with a common name. import urllib import urllib2 from HTMLParser import HTMLParser #from html.parser import HtmlLib from ProdSampleRequest import PhysicsGroup from ProdSample import ProdSample from TaskName import TaskName ################################################################################ class ProdSubmitterException: def __init__(self, msg): self.msg = msg def __str__(self): return self.msg ################################################################################ class ProdSubmitHtmlParser(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.dbg = 0 self.task_table_header = "Check TASK PARAMETERS and submit your request" self.found_task_table_header = False self.task_input_names = [] self.task_inputs = {} def handle_starttag(self, tag, attrs): sdbg = "ProdSubmitHtmlParser:handle_starttag: " if tag == "input": if self.dbg>2: print " Input attributes:" name = "" value = "" type = "" checked = False for vals in attrs: if self.dbg>2: print " " + str(vals) if vals[0] == "type": type = vals[1] elif vals[0] == "name": name = vals[1] elif vals[0] == "value": value = vals[1] elif vals[0] == "checked": checked = True radio = (type == "radio") keep = False if name == "mode": keep = True if self.found_task_table_header: if not radio: keep = True if radio and checked: keep = True if keep: if self.dbg>1: print sdbg + " Keeping: " + name + ": " + value self.task_input_names += [name] self.task_inputs[name] = value else: if self.dbg>1: print sdbg + " Skipping: " + name + ": " + value def handle_data(self, data): sdbg = "ProdSubmitHtmlParser:handle_data: " if data.strip() == self.task_table_header: if self.dbg>1: print sdbg + "Found table header" self.found_task_table_header = True ################################################################################ # ProdSubmitter constructs a job submission URL and then submits it: # sub = ProdSubmitter(user, group, intask, tag, nevt, fmt) # sub.get_url_submit() # sub.submit() class ProdSubmitter: def __init__(self, user, group, intask, tag, nevent =0, data_format =""): self.dbg = 0 self.user = user self.group = group self.prodgroup = PhysicsGroup(group).prodname() if isinstance(intask, str): self.intask = TaskName(intask) else: self.intask = intask self.tag = tag self.project = "mc08" self.data_format = data_format self.nevent = nevent self.url_panda = "http://panda.cern.ch:25980/server/pandamon/query//" self.urldata_names = [] # Increase key was added May 13: set use_increase_key to use this key. # 0 - Ignore key # 1 - Read and increment key # 2 - Set key to 0 self.use_increase_key = 2 self.urldata_names1 = ["mode", "qFormats", "qDSFFile", "qDSNFiles", "qTRFCache", "pList"] #self.urldata_names += ["qJO1", "qJO2", "qJO3", "qJO4", "qJO5" self.urldata_names2 = [ "qTaskREvents", "qTaskNEvents", "qTaskFEvents"] self.urldata_names2 += ["qPriority", "qTaskGrid", "qProjectMode", "qDSPhLong"] self.urldata_names2 += ["qphysgroup"] if self.use_increase_key == 1: self.urldata_names2 += ["qJO666"] elif self.use_increase_key == 2: self.urldata_names2 += ["qJO665"] self.urldata_names2 += ["task_request_email", "qProject", "qDSInput", "qTaskName"] self.urldata_names2 += ["qTaskTRF", "qTaskTRFVersion", "qSWRelease", "qMail", 'qCPUJob', 'qRAMJob'] self.evtperfile = {} self.evtperfile["r635"] = 10000 self.evtperfile["r641"] = 5000 self.evtperfile["r642"] = 10000 self.evtperfile["r643"] = 5000 self.url_confgure = "" self.url_submit = "" self.ready_to_submit = False self.html_configure_response = "Request not configured" self.html_configure_response += "
Input task: " + self.intask.name self.html_configure_response += "
Tag: " + self.tag + "" self.html_submit_response = "Request not submitted" self.html_submit_response += "
Input task: " + self.intask.name self.html_submit_response += "
Tag: " + self.tag + "" self.text_submit_response = "" self.html_configure_response_file = "" self.nevent_available = -1 # events available for processing def version(self): return "1.0.0" def get_url_configure(self): if len(self.url_confgure): return self.url_configure inputs={} inputs['mode']='reqtask1' inputs['qProject'] = self.project inputs['qDSInput'] = self.intask inputs['qTaskTRF']='modgen' inputs['qTaskTRFVersion']='14.2.25.6' inputs['qProjectMode'] = self.tag inputs['qMail'] = self.user inputs['action']='Continue' url_values = urllib.urlencode(inputs) self.url_configure = full_url = self.url_panda + '?' + url_values return self.url_configure def get_url_submit(self): if len(self.url_submit): return self.url_submit url_confgure = self.get_url_configure() req = urllib2.Request(url_confgure) response = urllib2.urlopen(req) self.html_configure_response = response.read() if len(self.html_configure_response_file): file = open(self.html_configure_response_file, "w") file.write(self.html_configure_response) file.close() #print self.html_configure_response # Fill urldata from the web page. self.urldata = {} use_HTMLParser = True if use_HTMLParser: #newhtml = self.html_configure_response.replace("@", "©") newhtml = self.html_configure_response.replace("@", "%40") newhtml = newhtml.replace("1: print "**** " + name + ": " + value self.urldata[name.encode("utf-8")]=value.encode("utf-8") # Add fields required for all transforms. # Missing because above does not handle radio buttons or menu. self.urldata["qTaskGrid"] = "panda" self.urldata["qphysgroup"] = self.prodgroup # Allow user to override the number of events. if self.nevent > 0: nfile_old = int(self.urldata.get("qDSNFiles", 0)) nevent_old = int(self.urldata.get("qTaskNEvents",0)) self.nevent_available = nevent_old nevent = self.nevent if nevent > nevent_old: raise ProdSubmitterException("get_url_submit: More events requested than available: " + str(nevent) + "/" + str(nevent_old)) nfile = (nevent*nfile_old)/nevent_old self.urldata["qTaskNEvents"] = str(nevent) self.urldata["qDSNFiles"] = str(nfile) # Transformation-specific fields. chtag = self.tag[0] if chtag == "t": tname = TaskName(self.intask) tag = tname.last_tag() #print "Input tag is ",tag if not self.evtperfile.has_key(tag): raise ProdSubmitterException("get_url_submit: Unable to find merge output event count for tag " + tag) self.urldata["qTaskFEvents"] = self.evtperfile[tag] self.urldata["qPriority"] = 550 elif chtag == "r": self.urldata["qPriority"] = 500 if len(self.data_format) == 0: raise ProdSubmitterException("get_url_submit: Reco tag requires data format") if self.data_format not in [ "AOD", "ESD", "AOD.ESD" ]: raise ProdSubmitterException("get_url_submit: Invalid data format: " + self.data_format) self.urldata["qFormats"] = self.data_format else: raise ProdSubmitterException("get_url_submit: Invalid tag: " + self.tag) if self.dbg: self.show_submit_data() # Increment the increase key. if self.use_increase_key: if self.use_increase_key == 1: keykey = "qJO666" if keykey in self.urldata: keyval = int(self.urldata[keykey]) keyval += 1 self.urldata[keykey] = str(keyval) else: raise ProdSubmitterException("get_url_submit: increase key field " + keykey + " not found") elif self.use_increase_key == 2: keykey = "qJO665" self.urldata[keykey] = "0" else: raise ProdSubmitterException("get_url_submit: invalid value for use_increase_key: " + str(use_increase_key)) # Build the ordered list of option names. self.urldata_names += self.urldata_names1 for iopt in range(0,99): name = "qJO" + "%i"%iopt if name in self.urldata: self.urldata_names += [name] self.urldata_names += self.urldata_names2 # Construct the submission URL. url_values_default = urllib.urlencode(self.urldata) if len(self.urldata_names): url_values = "" for name in self.urldata_names: if len(url_values): url_values += "&" if name not in self.urldata: raise ProdSubmitterException("get_url_submit: URL data does not include " + name) value = str(self.urldata[name]) value = value.replace("@", "%40") value = value.replace(",", "%2C") value = value.replace(";", "%3B") value = value.replace(" ", "+") url_values += name url_values += "=" url_values += value else: url_values = url_values_default self.url_submit = self.url_panda + "?" + url_values self.ready_to_submit = True return self.url_submit # Compare the submission URL with an input string. def check_url_submit(self, url_check): if ( url_check == self.url_submit ): print "URL's are identical" else: lcheck = len(url_check) lsubmit = len(self.url_submit) for ich in range(0, min(lcheck, lsubmit)): ccheck = url_check[ich] csubmit = self.url_submit[ich] if ccheck != csubmit: print "URL's differ at character " + str(ich) print "ASCII codes submit/reference: " + str(ord(csubmit)) + "/" + str(ord(ccheck)) print " Reference: ..." + url_check[ich:min(ich+60,lcheck)] print " Submit: ..." + self.url_submit[ich:min(ich+60,lsubmit)] break print "URL lengths submit/reference: " + str(lsubmit) + "/" + str(lcheck) # Show the map used to build the URL. def show_submit_data(self): for name in self.urldata: print "%20s: %-s"%(name,str(self.urldata[name])) # Return the nominal number of events per output file. def events_per_file(self): return int(self.urldata.get("qTaskFEvents", 0)) # Return the total number of events to be processed. def event_count(self): return int(self.urldata.get("qTaskNEvents", 0)) # Return the Prodsys name for the physics group. def physics_group_name(self): return self.urldata["qphysgroup"] # Return the number of input files. def input_file_count(self): return self.urldata.get("qDSNFiles", -1) # Return the number of ther first input file. def first_input_file(self): return self.urldata.get("qDSFFile", -1) # Submit the task(s). def submit(self): self.text_submit_response = "exception in get_url_submit" self.get_url_submit() self.text_submit_response = "exception after get_url_submit" if not self.ready_to_submit: self.text_submit_response = "not ready to submit" return 1 if self.event_count() == 0: self.text_submit_response = "Event count is zero" return 2 if len(self.physics_group_name()) == 0: self.text_submit_response = "Invalid physics group: " + self.group return 3 req = urllib2.Request(self.url_submit) response2 = urllib2.urlopen(req) self.html_submit_response = response2.read() state, message = self.check_html_submit_response() if state == 0: self.text_submit_response = "submitted" else: self.text_submit_response = message return state # Parse the response and extract the response message. # We assume success will have a green colored message and failure # will have a tomato-colored message. def check_html_submit_response(self): message = "Unable to find HTML submit response" state = 4 if len(self.html_submit_response): ic_tomato = self.html_submit_response.find("tomato") ic_green = self.html_submit_response.find("green") ic1 = -1 ic2 = -1 if ic_tomato>0 and ic_green<0: state = 5 ic1 = self.html_submit_response.find(">", ic_tomato) elif ic_green>0 and ic_tomato<0: state = 0 ic1 = self.html_submit_response.find(">", ic_green) else: state = 6 if ic1 > 0: ic2 = self.html_submit_response.find("<", ic1) if ic2 > 0: message = self.html_submit_response[ic1+1:ic2] else: message = "Unable to parse HTML submit response" print "ProdSubmitter:check_html_submit_response: ", message return state, message.strip() ################################################################################