from __future__ import print_function # For Python 2.5 compatability. from future import standard_library standard_library.install_aliases() from builtins import range import os import re import string import sys import traceback import xmlrpc.client import getpass import glob import webbrowser import hutil.file import hutil.json import hqueue as hq # A simple regular expression to check emails _VALID_EMAIL = re.compile(r""".+@.+""") def submitJob(parms, submit_function): """Submits a job with the given parameters and function after checking to see if the project files need to be copied or not. submit_function will be passed parms """ if parms["hip_action"] == "copy_to_shared_folder": _copyToSharedFolder(parms, submit_function) else: submit_function(parms) def _copyToSharedFolder(parms, submit_function): """Copies the files to a shared folder and then submits the job using the given function. submit_function will be passed parms """ # Copy project files to the shared folder. # We don't want to keep the ROP's interrupt dialog open, # so we exit the calling SOHO script so that the dialog closes # and then we schedule an event to run the copy and render code # (which includes opening the file dependency dialog). import hdefereval import hou hq_rop = hou.pwd() hdefereval.executeDeferred(_copyAndThenSubmitJob, hq_rop, parms, submit_function) def _copyAndThenSubmitJob(hq_rop, parms, submit_function): """Simple callback which copies the project files to the shared folder and then submits a job to HQueue which simulates the project. submit_function will be passed parms """ success = copyProjectFilesToSharedFolder(hq_rop, parms) if not success: return submit_function(parms) def getOutputParm(rop_node): """Return the output file parameter from the given ROP node. Return None if `rop_node` is None or if it is not a recognized output driver. """ if rop_node is None: return None rop_type = rop_node.type().name() # Handle Fetch ROPs. # Recursively follow the Fetch ROP's source node. if rop_type == "fetch" and rop_node.parm("source"): fetched_rop_node = \ rop_node.node(rop_node.parm("source").evalAsString()) return getOutputParm(fetched_rop_node) # If we are pointing to an HQueue ROP, then what we really want # is the node specified by the Output Driver parameter. if rop_type == "hq_render" or rop_type == "hq_sim": rop_node = getOutputDriver(rop_node) if rop_node is None: return None rop_type = rop_node.type().name() rop_type = rop_type.split(':')[0] # We handle only a subset of ROP node types. output_parm = None if rop_type == "ifd": output_parm = rop_node.parm("vm_picture") elif rop_type == 'vray_renderer': output_parm = rop_node.parm('SettingsOutput_img_file_path') elif rop_type in ("rop_geometry", "geometry"): output_parm = rop_node.parm("sopoutput") elif rop_type in ("rop_comp", "comp"): output_parm = rop_node.parm("copoutput") elif rop_type in ("rop_dop", "dop"): output_parm = rop_node.parm("dopoutput") elif rop_type == "baketexture": # The Bake Texture ROP node has multiple output parameters so we just # choose the first one. output_parm = rop_node.parm("vm_uvoutputpicture1") elif rop_type == "filecache": output_parm = rop_node.parm("file") elif rop_type == "karma": output_parm = rop_node.parm("picture") return output_parm def getOutputParmPattern(rop_node): """Return the file pattern in the ROP node's output parameter. Return None if no output parameter could be found. """ output_parm = getOutputParm(rop_node) if output_parm is None: return None return getUnexpandedStringParmVal(output_parm) def getOutputDirVariable(rop_node): """Return the name of the Houdini variable that expands to the output directory of the given ROP node, without a trailing /. Return None if no valid output directory variable could be determined. """ # Check the files generated by the rop node to determine whether they're # using $HIP or $JOB. output_parm = getOutputParm(rop_node) if output_parm is None: return None output_pattern = getUnexpandedStringParmVal( output_parm.getReferencedParm()) return checkPath(output_pattern) def checkPath (path): """Returns the name ofthe Houdini variable that the given path starts with, without the trailing /. Returns None if no valid output directory variable could be determined. """ if not (path.startswith("$") or path.startswith("{")): return None variable = path[1:].split("/")[0] return (variable if variable in ("HIP", "JOB", "{HIP}", "{JOB}") else None) def getOutputDriver(rop_node): """Returns the node which does the rendering/simulation work. For eg, if a hq_sim ROP points to a wedge ROP which points to a Geometry ROP, then this function will return the Geometry ROP. This function skips over all "pointer" ROPs. Return None if the Output Driver parameter is empty or points to a node path that does not exist. Also return None `hq_rop_node` is None or if the Output Driver parameter does not exist. """ # These ROP types point to the # ROPs which do the actual rendering/sims. pointer_rops = ("hq_sim", "hq_render", "wedge", "fetch") if rop_node is None: return None if len(rop_node.inputs()) > 0: output_rop = rop_node.inputs()[0] output_rop_type = output_rop.type().name() if output_rop_type in pointer_rops: return getOutputDriver(output_rop) if output_rop is not None: return output_rop # A mapping from rop types to # the parm name of their output # driver names. Incomplete for now. driver_parm_names = { "hq_sim": "hq_driver", "hq_render": "hq_driver", "wedge": "driver", "fetch": "source", } rop_type = rop_node.type().name() output_driver_parm = None if rop_type in list(driver_parm_names.keys()): output_driver_parm = rop_node.parm( driver_parm_names[rop_type]) if output_driver_parm is None: return None output_rop = rop_node.node(output_driver_parm.eval()) if output_rop is None: return None output_rop_type = output_rop.type().name() # Making a recursive call until we # get the required type of output driver. if output_rop_type in pointer_rops: return getOutputDriver(output_rop) return output_rop def getOutputDriverString(rop_node): """Return the node set in the Output Driver parameter of 'rop node'. Returns None if the there is no Output Driver parameter. """ if len(rop_node.inputs()) > 0: output_driver = rop_node.inputs()[0] if output_driver is not None: return output_driver.path() rop_type = rop_node.type().name() if rop_type in ('hq_sim', 'hq_render'): return rop_node.parm('hq_driver').eval() elif rop_type in ('wedge',): return rop_node.parm('driver').eval() return None def getFrames(hq_rop_node): """Return a tuple of frame numbers that the given HQueue ROP node is configured to render.""" import hou # We grab the frame range from the output driver # specified by the HQueue ROP node. output_driver = getOutputDriver(hq_rop_node) if output_driver is None: return (int(hou.frame()),) if output_driver.parm("trange").evalAsString() == "off": return (int(hou.frame()),) start, end, increment = ( int(x) for x in output_driver.parmTuple("f").eval()) return tuple(range(start, end+1, increment)) def getFileInfosFromFileReferences(file_references, hq_rop_node): """Return a sequence of file_info dictionaries corresponding to the files matching the given file references. """ # For each file pattern, we store a list of files that match that # pattern. We use a separate list for the file infos instead of using # file_path_to_file_info.values() to preserve the order. file_path_to_file_info = {} file_infos = [] frames = getFrames(hq_rop_node) for file_reference in file_references: for file_path in _expandFileReference(file_reference, frames): # If we've already encountered this file or if it doesn't exist # then skip it. if (file_path in file_path_to_file_info or not os.path.isfile(file_path)): continue file_info = { "path": file_path, "size": os.path.getsize(file_path), "checksum": hutil.file.checksum(file_path), } file_path_to_file_info[file_path] = file_info file_infos.append(file_info) return file_infos def _expandFileReference(file_reference, frames): import hou file_paths = set() result = [] for frame in frames: source_parm = file_reference[0] file_pattern = file_reference[1] if source_parm is not None: # If the file reference is from a parameter then it is better to just # evaluate the parameter because it handles channel references and # context-specific variables like $OS. file_path = source_parm.evalAsStringAtFrame(frame) else: file_path = hou.expandStringAtFrame(file_pattern, frame) if file_path not in file_paths: for filename in glob.glob(file_path): result.append(filename) file_paths.add(file_path) return result def getBaseParameters(): """Return a dictionary of the base parameters used by all HQueue ROPs.""" import hou parms = { "name" : hou.ch("hq_job_name").strip(), "assign_to" : hou.parm("hq_assign_to").evalAsString(), "clients": hou.ch("hq_clients").strip(), "client_groups" : hou.ch("hq_client_groups").strip(), "dirs_to_create": getDirectoriesToCreate(hou.pwd(), expand=False), "environment" : getEnvVariablesToSet(hou.pwd()), "hfs": getUnexpandedStringParmVal(hou.parm("hq_hfs")), "hq_server": hou.ch("hq_server").strip(), "open_browser": hou.ch("hq_openbrowser"), "priority": hou.ch("hq_priority"), "hip_action": hou.ch("hq_hip_action"), "autosave": hou.ch("hq_autosave"), "warn_unsaved_changes": hou.ch("hq_warn_unsaved_changes"), "report_submitted_job_id": hou.ch("hq_report_submitted_job_id"), "skip_file_dependecy_dialog": hou.ch("hq_skip_file_dependency_dialog"), } _addSubmittedByParm(parms) # Get the .hip file path. if parms["hip_action"] == "use_current_hip": parms["hip_file"] = hou.hipFile.path() elif parms["hip_action"] == "use_target_hip": parms["hip_file"] = getUnexpandedStringParmVal(hou.parm("hq_hip")) elif parms["hip_action"] == "copy_to_shared_folder": # Get the target project directory from the project path parameter. # We need to resolve $HQROOT if it exists in the parameter value. # To do that, we temporarily set $HQROOT, evaluate the parameter, # and then unset $HQROOT. old_roots = {} env_vars = getNetworkFolderVars(parms["hq_server"]) for var in env_vars: old_roots[var] = hou.hscript("echo $%s" % var)[0].strip() hou.hscript("set %s=%s" % (var, getEnvRoot(parms["hq_server"], var))) target_dir = hou.parm("hq_project_path").eval().strip() for var in env_vars: if old_roots[var] != "": hou.hscript("set %s=%s" % (var, old_roots[var])) else: hou.hscript("set -u %s" % var) if target_dir[-1] != "/": target_dir = target_dir + "/" # Set the target .hip file path. parms["hip_file"] = target_dir + os.path.basename(hou.hipFile.name()) # When copying to the shared folder, # we always create a new .hip file. parms["hip_file"] = _getNewHipFilePath(parms["hip_file"]) else: pass # Setup email parameters if hou.ch("hq_will_email"): # We remove the whitespace around each email entry parms["emailTo"] = ",".join([email.strip() for email in hou.ch("hq_emailTo").split(',')]) # The values added to parms for why emails should be sent are # place holders. # When jobSend is ran they are updated as we need a server connection # to add the values we want. email_reasons = [] if hou.ch("hq_email_on_start"): email_reasons.append("start") if hou.ch("hq_email_on_success"): email_reasons.append("success") if hou.ch("hq_email_on_failure"): email_reasons.append("failure") if hou.ch("hq_email_on_pause"): email_reasons.append("pause") if hou.ch("hq_email_on_resume"): email_reasons.append("resume") if hou.ch("hq_email_on_reschedule"): email_reasons.append("reschedule") if hou.ch("hq_email_on_priority_change"): email_reasons.append("priority change") parms["emailReasons"] = email_reasons else: parms["emailTo"] = "" # An empty list for emailReasons means no email will be sent parms["emailReasons"] = [] return parms def _addSubmittedByParm(parms): """Adds who submits the job to the base parameters.""" try: parms["submittedBy"] = getpass.getuser() except (ImportError, KeyError): pass def _getNewHipFilePath(hip_file_path): """Return a new .hip file path using `hip_file_path` as the base file path. """ count = 1 base_path, extension = os.path.splitext(hip_file_path) new_path = hip_file_path # Keep incrementing the file suffix until # we have a file name that does not exist. while True: new_path = base_path + "_" + str(count) + extension if not os.path.exists(new_path): break count = count + 1 return new_path def getDirectoriesToCreate(hq_node, expand=True): """Return a list of directory paths that should be created on the farm. The result is based on the contents of the "Create Directories" parameter. If `expand` is True, then return a list of directories with all variables expanded. """ dirs_parm = hq_node.parmTuple("create_directories") if dirs_parm is None: return [] num_dirs = dirs_parm.eval()[0] dir_paths = [] for i in range(0, num_dirs): path_parm = hq_node.parm("directory_path" + str(i)) if path_parm is None: continue if expand: path = path_parm.eval().strip() else: path = getUnexpandedStringParmVal(path_parm) if path == "": continue dir_paths.append(path) return dir_paths def getEnvVariablesToSet(hq_node): """Return a dictionary of environment variables to set on the client machine when processing a job.""" env_parm = hq_node.parmTuple("environment") if env_parm is None: return {} num_vars = env_parm.eval()[0] env = {} for i in range(0, num_vars): name_parm = hq_node.parm("var_name" + str(i)) value_parm = hq_node.parm("var_value" + str(i)) if name_parm is None or value_parm is None: continue name = name_parm.eval().strip() value = value_parm.eval().strip() env[name] = value return env def checkBaseParameters(parms): """Check the values of the given parameters. Return True if all the parameter values are valid and False otherwise. """ # Check HQ Server. if parms["hq_server"] == "": displayError("HQueue Server cannot be blank.") return False # Check remote .hip file path. if (parms["hip_action"] == "use_target_hip" or parms["hip_action"] == "copy_to_shared_folder"): if parms["hip_file"] == "": displayError("Target HIP cannot be blank.") return False # Check remote HFS path. if parms["hfs"] == "": displayError("Target HFS cannot be blank.") return False # Check Clients parameter. if parms["assign_to"] == "clients" and parms["clients"] == "": displayError("Clients parameter cannot be blank.") return False # Check Client Groups parameter. if parms["assign_to"] == "client_groups" and parms["client_groups"] == "": displayError("Client Groups parameter cannot be blank.") return False # Check environment variables. errors = "" for var_name, var_value in list(parms["environment"].items()): if var_name == "": errors += "Environment variable name cannot be blank.\n" elif re.match("^[a-zA-Z_][a-zA-Z_0-9]*$", var_name) is None: errors += "Bad environment variable name '%s'.\n" % var_name if errors != "": displayError(errors) return False # Check email is valid if emails are to be sent if parms["emailReasons"]: invalid_emails = _getInvalidEmails(parms["emailTo"]) if invalid_emails: error_message = ("The following emails are invalid: \n" + "\n".join(invalid_emails)) displayError(error_message) return False return True def checkOutputDriver(output_driver_string, _visited_nodes = None): import hou if _visited_nodes is None: _visited_nodes = set() # Check output driver. if output_driver_string == "": displayError("Output Driver cannot be blank.") return False # Check if the driver exists. output_driver = hou.node(output_driver_string) if output_driver is None: displayError("%s does not exist." % output_driver_string) return False _visited_nodes.add(output_driver.path()) target_output_driver_string = getOutputDriverString(output_driver) if target_output_driver_string is not None: if target_output_driver_string not in _visited_nodes: return checkOutputDriver(target_output_driver_string, _visited_nodes) if output_driver.inputs(): for to_visit_node in output_driver.inputs(): if to_visit_node is not None: to_visit_path = to_visit_node.path() if to_visit_path not in _visited_nodes: if not checkOutputDriver(to_visit_path, _visited_nodes): return False return True def _getInvalidEmails(email_string): """Returns the list of invalid emails in a string containing a comma seperated list of emails. This is not very strict in wht it considers an invalid email but it will not wrongly say an email is invalid. """ emails = email_string.split(',') invalid_emails = [] for email in emails: if not (email and _VALID_EMAIL.match(email)): invalid_emails.append(email) return invalid_emails def displayError(msg, exception=None): """Pop up a message dialog to display the given error message. If the ui is unavailable, then it writes the message to the console. """ import hou if hou.isUIAvailable(): details = (str(exception) if exception is not None else None) hou.ui.displayMessage(msg, severity=hou.severityType.Error, details=details) else: if exception is not None: msg += "\n" + str(exception) raise hou.OperationFailed(msg) def displayMessage(msg): """Pop up a message dialog to display the given message. If the ui is unavailable, then it writes the message to the console. """ import hou if hou.isUIAvailable(): hou.ui.displayMessage(msg, severity=hou.severityType.Message) else: print(msg) def warnOrAutoSaveHipFile(parms): """Automatically save the current .hip file or display a warning to the user about unsaved changes. This function is meant to be called before an HQueue job submission as a way of informing the user about unsaved changes that may not trickle down to the HQueue farm. Return True if the job submission should continue on. Return False if the user chose to cancel the job submission. """ import hou # Automatically save the .hip file if the autosave parameter # is turned on. We never auto-save if the user is rendering # a target .hip file. if parms["hip_action"] != "use_target_hip" and parms["autosave"]: hou.hipFile.save() return True # We don't need to check for unsaved changes if we are # rendering a target .hip file anyway. if parms["hip_action"] == "use_target_hip": return True # Check if we care about unsaved changes. if not parms["warn_unsaved_changes"]: return True # Check for unsaved changes. if not hou.hipFile.hasUnsavedChanges(): return True # Pop-up a warning about the unsaved changes. # Give the user the choice of either continuing or cancelling. should_continue = True msg = "WARNING: Your .hip file has unsaved changes." if hou.isUIAvailable(): msg = msg + "\n\n" + "Submit the HQueue job anyway?" button_index = hou.ui.displayMessage(msg, buttons=("Yes", "No")) should_continue = (button_index == 0) else: msg = msg + " Submitting HQueue job anyway." print(msg) return should_continue def sendJob(hq_server, main_job, open_browser, report_submitted_job_id): """Send the given job to the HQ server. If the ui is available, either display the HQ web interface or display the id of the submitted job depending on the value of `open_browser` and 'report_submitted_job_id'. """ import hou s = _connectToHQServer(hq_server) if s is None: return False # We do this here as we need a server connection _setupEmailReasons(s, main_job) # If we're running as an HQueue job, make that job our parent job. try: ids = s.newjob(main_job, os.environ.get("JOBID")) except Exception as e: displayError("Could not submit job to '" + hq_server + "'.", e) return False # Don't open a browser window or try to display a popup message if we're # running from Houdini Batch. if not hou.isUIAvailable(): return True jobId = ids[0] if not open_browser and report_submitted_job_id: buttonindex = hou.ui.displayMessage("Your job has been submitted (Job %i)." % jobId, buttons=("Open HQueue", "Ok"), default_choice=1 ) if buttonindex == 0: open_browser = True if open_browser: url = "%(hq_server)s" % locals() webbrowser.open(url) return True def _setupEmailReasons(server_connection, job_spec): """Changes the placeholder string in the emailReasons part of the job spec with the actual string that will be sent. Gives a warning message if any of the options do not exist on the server side. """ import hou placeholder_reasons = job_spec["emailReasons"] new_reasons = [] failure_messages = [] for reason in placeholder_reasons: try: if reason == "start": new_reasons.extend( server_connection.getStartedJobEventNames()) elif reason == "success": new_reasons.extend(server_connection.getSucceededStatusNames()) elif reason == "failure": new_reasons.extend(server_connection.getFailedJobStatusNames()) elif reason == "pause": new_reasons.extend(server_connection.getPausedJobStatusNames()) elif reason == "resume": new_reasons.extend(server_connection.getResumedEventNames()) elif reason == "reschedule": new_reasons.extend( server_connection.getRescheduledEventNames()) elif reason == "priority change": new_reasons.extend( server_connection.getPriorityChangedEventNames()) else: raise Exception("Did not recieve valid placeholder reason " + reason + ".") except xmlrpc.client.Fault: failure_messages.append(string.capwords(reason)) if failure_messages: if hou.isUIAvailable(): base_failure_message = ("The server does not support sending " + "email for the following reasons:") failure_message = "\n".join(failure_messages) failure_message = "\n".join([base_failure_message, failure_message]) hou.ui.displayMessage(failure_message, severity = hou.severityType.Warning) job_spec["emailReasons"] = ",".join(new_reasons) def getJobCommands(hq_cmds, cmd_key, script=None): """Return platform-specific job commands defined by `cmd_key` and `script`. Return a dictionary where the keys are the supported platforms (i.e. linux, windows, macosx) and the values are the shell commands to be executed for the job. `cmd_key` is a key into the `hq_cmds` dictionary and should be either "hythonComands", "pythonCommands" or "mantraCommands". The optional `script` argument indicates whether the job should execute a script file. If `script` is None, then the job should run the commands from hq_cmds[cmd_key] as-is. """ linux_cmd_key = cmd_key + "Linux" win_cmd_key = cmd_key + "Windows" macosx_cmd_key = cmd_key + "Macosx" # Get Linux commands. Default to platform-independent commands # if Linux commands do not exist. linux_cmds = hq_cmds[linux_cmd_key] if linux_cmd_key in hq_cmds \ else hq_cmds[cmd_key] # Get Windows commands. Default to platform-independent commands # if Windows commands do no exist. win_cmds = hq_cmds[win_cmd_key] if win_cmd_key in hq_cmds \ else hq_cmds[cmd_key] # Get MacOSX commands. Default to platform-independent commands # if MacOSX commands do no exist. macosx_cmds = hq_cmds[macosx_cmd_key] if macosx_cmd_key in hq_cmds \ else hq_cmds[cmd_key] if script is not None: # Get the HQueue scripts directory. linux_hq_scripts_dir = hq.houdini.hqScriptsDirPath() macosx_hq_scripts_dir = hq.houdini.hqScriptsDirPath() win_hq_scripts_dir = hutil.file.convertToWinPath(linux_hq_scripts_dir, var_notation="!") linux_cmds = "%s %s/%s" % (linux_cmds, linux_hq_scripts_dir, script) macosx_cmds = "%s %s/%s" % (macosx_cmds, macosx_hq_scripts_dir, script) win_cmds = "%s \"%s\\%s\"" % (win_cmds, win_hq_scripts_dir, script) commands = { "linux" : linux_cmds, "windows" : win_cmds, "macosx" : macosx_cmds } return commands def getHipFileAndHFS(parms): """Return the hip file and hfs paths, prefixed with a network folder variable if possible.""" # Substitute the beginning of the .hip path with a network folder # variable (if applicable). hq_server = parms["hq_server"] hip_file = substituteWithFolderVar(hq_server, parms["hip_file"]) # Substitute the beginning of the path to $HFS with a network folder # variable (if applicable). hfs = substituteWithFolderVar(hq_server, parms["hfs"]) return hip_file, hfs def getHFS(parms): """Returns the hfs paths, prefixed with folder variables if possible.""" hq_server = parms["hq_server"] return substituteWithFolderVar(hq_server, parms["hfs"]) def expandHQROOT(path, hq_server): """Return the given file path with instances of $HQROOT expanded out to the mount point for the HQueue shared folder root.""" # Get the HQueue root path. hq_root = getEnvRoot(hq_server, "HQROOT") if hq_root is None: return path expanded_path = path.replace("$HQROOT", hq_root) return expanded_path def getHQROOT(hq_server): """Query the HQueue server and return the mount point path to the HQueue shared folder root. Return None if the path cannot be retrieved from the server. """ # Identify this machine's platform. platform = sys.platform if platform.startswith("win"): platform = "windows" elif platform.startswith("linux"): platform = "linux" elif platform.startswith("darwin"): platform = "macosx" # Connect to the HQueue server. s = _connectToHQServer(hq_server) if s is None: return None try: # Get the HQ root. hq_root = s.getHQRoot(platform) except: displayError( "Could not retrieve $HQROOT from '" + hq_server + "'.") return None return hq_root def getEnvRoot(hq_server, name, is_client=False): """Query the HQueue server and return the mount point path to the root of the HQueue shared folder with the given name. Return None if the path cannot be retrieved from the server. """ # Identify this machine's platform. platform = sys.platform if platform.startswith("win"): platform = "windows" elif platform.startswith("linux"): platform = "linux" elif platform.startswith("darwin"): platform = "macosx" # Connect to the HQueue server. s = _connectToHQServer(hq_server) if s is None: return None try: # Get the network folder root. root = s.getEnvRoot(platform, name, is_client) except: displayError( "Could not retrieve $" + name + " from '" + hq_server + "'.") return None return root def getNetworkFolderVars(hq_server): # Identify this machine's platform. platform = sys.platform if platform.startswith("win"): platform = "windows" elif platform.startswith("linux"): platform = "linux" elif platform.startswith("darwin"): platform = "macosx" # Connect to the HQueue server. s = _connectToHQServer(hq_server) if s is None: return None try: # Get the network folder variables. vars_ = s.getNetworkFolderNames() except Exception as e: displayError( "Could not retrieve network folders from '" + hq_server + "'.", exception=e) return None return vars_ def substituteWithFolderVar(hq_server, file_path, expand_frame_variables=True): """Replace the beginning of the given path with "$HQROOT". Return the new path. If the original path is not under HQueue's shared network file system, return the original path. Return None if an error occurs. """ if hq_server is None: return None names = getNetworkFolderVars(hq_server) if names is None: return None max_len = 0 final_file_path = None # Normalize file path. # Stick with forward slashes instead of backward slashes. file_path = os.path.normpath(file_path) file_path = file_path.replace("\\", "/") file_path = substituteNonHQVariables(file_path, expand_frame_variables) for name in names: folder_path = getEnvRoot(hq_server, name) if folder_path is None: continue # Stick with forward slashes instead of backward slashes. folder_path = os.path.normpath(folder_path) folder_path = folder_path.replace("\\", "/") if not file_path.startswith(folder_path) \ and sys.platform == "win32": # On Windows, we additionally check against the UNC path equivalent # of the network folder. folder_path = getEnvRoot(hq_server, name, True) if folder_path is None: continue folder_path = os.path.normpath(folder_path) folder_path = folder_path.replace("\\", "/") # Compare the network folder mount with the given file path. # Substitute the raw mount path with the folder environment # variable. if file_path.startswith(folder_path): common_seq = file_path[:len(folder_path)] if len(common_seq) < max_len: continue tmp_file_path = file_path[len(folder_path):] if not tmp_file_path.startswith("/"): tmp_file_path = "/" + tmp_file_path final_file_path = "$" + name + tmp_file_path if final_file_path is None: final_file_path = file_path return final_file_path def substituteNonHQVariables(file_path, expand_frame_variables=True): """Returns the file path with all the variables that are not HQ specfic expanded. """ import hou split_path = file_path.split("$") if split_path: new_path = [] # If a path starts with a variable than the first element of the # split_path list will be empty # Otherwise the first component is not a variable if not split_path[0]: split_path.pop(0) else: new_path.append(split_path.pop(0)) for component in split_path: # Do not expand variables that start with $HQ. if component.startswith("HQ"): new_path.append("$" + component) # Do not expand frame variables unless requested. elif (not expand_frame_variables and (component.startswith("F") or component.startswith("{F"))): new_path.append("$" + component) # Expand all other variables. else: new_path.append(hou.expandString("$" + component)) return "".join(new_path) else: return file_path def buildContainingJobSpec(job_name, hq_cmds, parms, child_job, apply_conditions_to_children=True): """Return a job spec that submits the child job and waits for it. The job containing the child job will not run any command. """ job = { "name": job_name, "priority": child_job["priority"], "environment": {"HQCOMMANDS": hutil.json.utf8Dumps(hq_cmds)}, "command": "", "children": [child_job], "emailTo": parms["emailTo"], "emailReasons": parms["emailReasons"], "triesLeft": parms.get("tries_left") } if "submittedBy" in parms: job["submittedBy"] = parms["submittedBy"] # Add job assignment conditions if any. conditions = { "clients":"host", "client_groups":"hostgroup" } for cond_type in list(conditions.keys()): job_cond_keyword = conditions[cond_type] if parms["assign_to"] == cond_type: job[job_cond_keyword] = parms[cond_type] if apply_conditions_to_children: for child_job in job["children"]: child_job[job_cond_keyword] = parms[cond_type] return job def getHQueueCommands(remote_hfs, num_cpus=0): """Return the dictionary of commands to start hython, Python, and mantra. Return None if an error occurs when reading the commands from the HQueueCommands file. If `num_cpus` is greater than 0, then we add a -j option to each command so that the application is run with a maximum number of threads. """ import hou # HQueueCommands will exist in the Houdini path. cmd_file_path = hou.findFile( "soho/python%d.%d/HQueueCommands" % sys.version_info[:2]) hq_cmds = {} cmd_file = open(cmd_file_path, "r") cmd_name = None cmds = None continue_cmd = False for line in cmd_file: line = line.strip() # Check if we need to continue the current command. if continue_cmd: # Add line to current command. cmds = _addLineToCommands(cmds, line) if line[-1] == "\\": continue_cmd = True else: cmds = _finalizeCommands(cmd_name, cmds, remote_hfs, num_cpus) if cmds is None: return None hq_cmds[cmd_name] = cmds continue_cmd = False continue # Ignore comments and empty lines. if line.startswith("#") or line.strip() == "": continue # Ignore lines with no command assignment. eq_op_loc = line.find("=") if eq_op_loc < 0: continue # Start a new command. cmd_name = line[0:eq_op_loc].strip() cmds = None line = line[eq_op_loc+1:] # Add line to current command. cmds = _addLineToCommands(cmds, line) if line[-1] == "\\": continue_cmd = True else: cmds = _finalizeCommands(cmd_name, cmds, remote_hfs, num_cpus) if cmds is None: return None hq_cmds[cmd_name] = cmds continue_cmd = False return hq_cmds def _addLineToCommands(cmds, line): """Adds the given line to the command string. This is a helper function for getHQueueCommands. """ line = line.strip() if line[-1] == "\\": line = line[:-1].strip() if cmds is None: cmds = line else: cmds = cmds + " " + line return cmds def _finalizeCommands(cmd_name, cmds, remote_hfs, num_cpus): """Perform final touches to the given commands. This is a helper function for getHQueueCommands. """ if cmd_name.endswith("Windows"): remote_hfs = hutil.file.convertToWinPath(remote_hfs, var_notation="!") # Replace HFS instances. cmds = cmds.replace("%(HFS)s", remote_hfs) # Attempt to replace other variable instances with # environment variables. try: cmds = cmds % os.environ except KeyError as e: displayError("Use of undefined variable in " + cmd_name + ".", e) return None # Strip out wrapper quotes, if any. # TODO: Is this needed still? if (cmds[0] == "\"" and cmds[-1] == "\"") \ or (cmds[0] == "'" and cmds[-1] == "'"): cmds = cmds[1:] cmds = cmds[0:-1] # Add the -j option to hython and Mantra commands. if num_cpus > 0 and ( cmd_name.startswith("hython") or cmd_name.startswith("mantra")): cmds += " -j" + str(num_cpus) return cmds def _setupHQServerProxy(hq_server): """Sets up a xmlrpclib server proxy to the given HQ server.""" if not hq_server.startswith("http://"): full_hq_server_path = "http://%s" % hq_server else: full_hq_server_path = hq_server return xmlrpc.client.ServerProxy(full_hq_server_path, allow_none=True) def _checkHQServerExists(server, hq_server): try: server.ping() return True except: displayError("Could not connect to '" + hq_server + "'.\n\n" + "Make sure that the HQueue server is running\n" + "or change the value of 'HQueue Server'.", TypeError("this is a type error")) return False def doesHQServerExists(hq_server): """Check that the given HQ server can be connected to. Returns True if the server exists and False if it does not. Furthermore, it will display an error message if it does not exists.""" server = _setupHQServerProxy(hq_server) return _checkHQServerExists(server, hq_server) def _connectToHQServer(hq_server): """Connect to the HQueue server and return the proxy connection.""" s = _setupHQServerProxy(hq_server) if _checkHQServerExists(s, hq_server): return s else: return None def selectClients(hq_server, clients_parm): """Select clients registered on the HQueue server from a list. A modal dialog pops-up displaying a list of all the client machines on the HQueue farm. The user selects the machines from the list and clicks on OK to populate the `clients_parm` parameter with the selection. """ import hou clients = _getClients(hq_server) if clients is None: return clients.sort() selection = hou.ui.selectFromList(clients, "") # Check if the user selected something. if len(selection) == 0: return client_string = "" for index in selection: if client_string != "": client_string = client_string + "," client_string = client_string + clients[index] clients_parm.set(client_string) def selectClientGroups(hq_server, client_groups_parm): """Select client groups on the HQueue server from a list. A modal dialog pops-up displaying a list of all the client groups on the HQueue farm. The user selects the groups from the list and clicks on OK to populate the `client_groups_parm` parameter with the selection. """ import hou client_groups = _getClientGroups(hq_server) if client_groups is None: return group_names = [] for client_group in client_groups: group_names.append(client_group["name"]) group_names.sort() selection = hou.ui.selectFromList(group_names, "") # Check if the user selected something. if len(selection) == 0: return group_string = "" for index in selection: if group_string != "": group_string = group_string + "," group_string = group_string + group_names[index] client_groups_parm.set(group_string) def _getClients(hq_server): """Return a list of all the clients registered on the HQueue server. Return None if the client list could not be retrieved from the server. """ s = _connectToHQServer(hq_server) if s is None: return None try: client_ids = None attribs = ["id", "hostname"] clients = s.getClients(client_ids, attribs) except: displayError("Could not retrieve client list from '" + hq_server + "'.") return None return [client["hostname"] for client in clients] def _getClientGroups(hq_server): """Return a list of all the client groups on the HQueue server. Return None if the client group list could not be retrieved from the server. """ s = _connectToHQServer(hq_server) if s is None: return None try: client_groups = s.getClientGroups() except: displayError("Could not retrieve client group list from '" + hq_server + "'.") return None return client_groups def getResources(hq_server): """Returns a list of dictionaries where each dictionary represents a resource. Return None if the resource list could not be retrieved from the server. """ s = _connectToHQServer(hq_server) if s is None: return None try: resources = s.getResources() except: displayError("Could not retrieve resource list from '" + hq_server + "'.") return None return resources def getHQueueServerMachineFromURL(hq_server_url): if hq_server_url.startswith("http://"): hq_server_url = hq_server_url[7:] return hq_server_url.split(":")[0] def getSelectedFileReferences(output_dir_variable): """Returns the required file references when the file dependency dialog is to be skipped. """ import hou required_dir_variable = "${0}".format( output_dir_variable) hip_file = hou.hipFile.path().replace( hou.getenv(output_dir_variable), required_dir_variable) hip_file_reference = (None, hip_file) selected_references = hou.fileReferences( output_dir_variable, include_all_refs=False) selected_references += (hip_file_reference,) return selected_references def copyProjectFilesToSharedFolder(hq_rop, parms): """Copy the render project's files to the target location specified in `parms`. Return True if the copy is successful. Return False otherwise. The user is first prompted by the file dependency dialog to choose the files to be copied to the shared folder. """ import hou # Get the target .hip file path and directory path. target_hip = parms["hip_file"] target_dir = os.path.dirname(target_hip) # The parameters structure needs to know about the new # .hip file name. parms["hip_file"] = target_hip # TODO: Get previously copied files. We need to send this # list to the file dependency dialog so that it can inform the # user which files do not have to be copied to the shared folder. # Actually, do we really need this? files_already_uploaded = [] # Need to examine the output driver and make sure # that it is writing to a valid file path. rop_node = getOutputDriver(hq_rop) if rop_node is None: displayError( "The Output Driver parameter does not point to an existing node.") return False output_dir_variable = getOutputDirVariable(rop_node) if output_dir_variable is None: displayError( ("Invalid output path specified in the output driver (%s).\n\n" + "Make sure that either $HIP or $JOB is used " + "in the output path.") % rop_node.path()) return False # Initialize other arguments that will be passed to the # file dependency dialog. patterns_to_unselect = (getOutputParmPattern(rop_node), ) is_standalone = True # Check whether the user wants to skip the Choose # File Dependency window. skip_file_dependecy_dialog = parms["skip_file_dependecy_dialog"] if skip_file_dependecy_dialog: selected_file_references = getSelectedFileReferences( output_dir_variable) else: # Pop-up the file dependency dialog so the user can choose # which files to copy to the shared folder. output_driver = getOutputDriver(hq_rop) pressed_ok, selected_file_references = \ hou.ui.displayFileDependencyDialog(hq_rop, files_already_uploaded, patterns_to_unselect, output_dir_variable, is_standalone) if not pressed_ok: return False file_infos = getFileInfosFromFileReferences( selected_file_references, hq_rop) # Start interruptable operation. with hou.InterruptableOperation( "Copying file", "Copying project files", True) as operation: # Iterate through each file and copy it to the shared folder. count = 0.0 HIP = hou.expandString("$HIP") success = True for file_info in file_infos: source_path = file_info["path"] dest_path = source_path.replace(HIP, target_dir, 1) # Update overall progress. overall_percent = float(count) / float(len(file_infos)) operation.updateLongProgress(overall_percent) # For the .hip file, copy it to the target .hip file path # chosen by the user. if source_path == hou.hipFile.path(): dest_path = target_hip success = _copyFileToSharedFolder(source_path, dest_path, file_info) if not success: break count = count + 1 return success def _copyFileToSharedFolder(source_path, dest_path, source_file_info): """Helper function for copyFilesToSharedFolder(). Copy the source file to the destination path. """ import hou dest_dir = os.path.dirname(dest_path) # Check if the destination file already exists. # If it does, then don't bother copying the source # if the destination has the same checksum and size. if os.path.exists(dest_path): dest_checksum = hutil.file.checksum(dest_path) dest_size = os.path.getsize(dest_path) if source_file_info["checksum"] == dest_checksum \ and source_file_info["size"] == dest_size: return True # Create the destination directory and subdirectories # if they do not exist. if not os.path.exists(dest_dir): try: os.makedirs(dest_dir) except: displayError( ("Failed to copy files to shared folder.\n\n" + "Could not create directory %s") % dest_dir) return False # Start interruptable operation. with hou.InterruptableOperation( "Copying %s" % os.path.basename(source_path)) as operation: # Copy the file to the shared folder. # We copy in chunks so that way we can report the progress # of the copy. success = True source_file = None dest_file = None try: # Open source and destination files. source_file = open(source_path, "rb") dest_file = open(dest_path, "wb") total_bytes = source_file_info["size"] read_bytes = 0 CHUNK_SIZE = 4096 while read_bytes < total_bytes: buffer = source_file.read(CHUNK_SIZE) read_bytes = read_bytes + len(buffer) dest_file.write(buffer) # Update the short progress. percent = float(read_bytes)/float(total_bytes) operation.updateProgress(percent) except Exception as e: displayError("Failed to copy %s to shared folder." % source_path, e) success = False finally: if source_file is not None: source_file.close() if dest_file is not None: dest_file.close() return success def getUnexpandedStringParmVal(parm): """Return the value for the given string parameter without expanding any variables. """ parm_val = "" # Get the source parameter in case the original parameter # was channel referencing another parameter. source_parm = parm.getReferencedParm() if len(source_parm.keyframes()) > 0: # Parameter has a keyframe. Just return the value parm_val = source_parm.eval() else: parm_val = source_parm.unexpandedString() return parm_val.strip() def checkForRecursiveChain(hq_node): """Checks to see if the HQueue node is an ancestor to its output driver.""" import hou import toolutils output_driver = hou.node(hq_node.parm('hq_driver').evalAsString()) return toolutils.findInputNode(output_driver, hq_node)