Location: PHPKode > projects > Huygens Remote Manager > hrm/inc/QueueManager.inc
<?php
// This file is part of the Huygens Remote Manager
// Copyright and license notice: see license.txt

require_once ("Setting.inc");
require_once ("Database.inc");
require_once ("JobDescription.inc");
require_once ("hrm_config.inc");
require_once ("Fileserver.inc");
require_once ("Shell.inc");
require_once ("Mail.inc");
require_once ("Job.inc");
require_once ("JobQueue.inc");

//!---------------------------------------------------------
// @class    QueueManager
// @desc     Translate job description to huygens script
//           and run it on a free server
//!---------------------------------------------------------

Class QueueManager {
	public $queue;
	public $freeServer;
	public $job;
	public $shallStop;
	// TODO make it a local variable
	public $stopTime;
	public $nping;

	function QueueManager() {
		$this->runningJobs = array ();
		$this->queue = new JobQueue();
		$this->shallStop = False;
		$this->nping = array();
	}

	function executeScript($job) {
		global $imageProcessingIsOnQueueManager;
		global $copy_images_to_huygens_server;
		global $logdir;
		
		$server = $this->freeServer;
		// server name without proc number
		$s = split(" ", $server);
		$server_hostname = $s[0];
		$desc = $job->description();
		$clientScriptPath = $desc->sourceFolder();
		$scriptName = $job->scriptName();
                report(">>>>> Executing script: $imageProcessingIsOnQueueManager $copy_images_to_huygens_server", 2);
		if (!$imageProcessingIsOnQueueManager && $copy_images_to_huygens_server) {
			$clientScriptPath = $this->copyImagesToServer($job, $server_hostname);
			report("images copied to IP server", 1);
		}
		$proc = newExternalProcessFor($server, $server . "_" .$job->id() . "_out.txt", $server . "_" .$job->id(). "_error.txt");
		report("shell process created", 1);
		$ssh = $proc->runShell();
		report("running shell: $clientScriptPath$scriptName", 1);
		$pid = $proc->runHuygensScript($clientScriptPath . $scriptName);
		// TODO refactor this <<
		// check if Huygens outputs any interactive dialog
		# sleep(5);

                // This keeps the queue manager blocked until the job is
                // completed. I guess that's not the intention! I'll deactivate
                // it by now.
                /*
                while (True) {
			if (feof($proc->out_file)) {
                            report ("EOF ".$proc->out_file.": ", 1);
				break;
                        }
			$input = fgets($proc->out_file);

                        // Why is this error processing necessary here?
			if (strstr($input, "Microscope types of original and PSF differ:")) {
				$warning_file = fopen($logdir . "/" . $server . "_warning.txt", "a");
				fwrite($warning_file, "[" . $desc->id() . "] Microscope types of original and PSF differ\n");
				fclose($warning_file);
				break;
			}
			else if (strstr($input, "After necessary resampling the PSF is too small.")) {
				$warning_file = fopen($logdir . "/" . $server . "_warning.txt", "a");
				fwrite($warning_file, "[" . $desc->id() . "] After necessary resampling the PSF is too small\n");
				fclose($warning_file);
				break;
			}
		}
                */
		// >>
		report("running script (pid $pid)", 1);

                // Release the shell, the script in the background will keep
                // running.
                $proc->release();

                # $this->activeProcs[$job->id()] = $proc;
		$job->setPid($pid);
		$job->setServer($server);
		return ($pid > 0);
	}
	
 	function copyImagesToServer($job, $server_hostname) {
		global $huygens_user;
		global $image_folder;
		global $image_source;
		global $image_destination;
		global $huygens_server_image_folder;

		report("Copying images to IP server", 2);
		
                $desc = $job->description();
		$user = $desc->owner();
		
		// TODO substitute spaces by underscores in image name to avoid processing problems with Huygens
		
		$batch = "cd \"" . $huygens_server_image_folder . "\"\n";
		$batch .= "-mkdir \"" . $user->name() . "\"\n";
		$batch .= "cd \"" . $user->name() . "\"\n";
		$batch .= "-mkdir \"" . $image_source . "\"\n";
		$batch .= "cd \"" . $image_source . "\"\n";
		$batch .= "put \"" . $image_folder . "/" . $user->name() . "/" . $image_source . "/" . $job->scriptName() . "\"\n";
		
		// Transfer the experimental PSF(s) 
		$parameterSetting = $desc->parameterSetting;
		$psfParam = $parameterSetting->parameter('PointSpreadFunction');
		if ($psfParam->value() == "measured") {
			$psf = $parameterSetting->parameter('PSF');
			$values = $psf->value();
			foreach ($values as $value) {
				$path = split("/", $value);
				if (sizeof($path) > 0) {
					for ($i = 0; $i < sizeof($path) - 1; $i++) {
						$batch .= "-mkdir \"" . $path[$i] . "\"\n";
						$batch .= "cd \"" . $path[$i] . "\"\n";
					}
				}
				$filename = $image_folder . "/" . $user->name() . "/" . $image_source . "/" . $value;
				if (stristr($filename, ".ics"))  {
					$batch .= "put \"" . $filename . "\"\n";
					$filename = eregi_replace(".ics", ".ids", $filename);
					$batch .= "put \"" . $filename . "\"\n";
				}
				else {
					$batch .= "put \"" . $filename . "\"\n";
				}
				if (sizeof($path) > 0) {
					for ($i = 0; $i < sizeof($path) - 1; $i++) {
						$batch .= "cd ..\n";
					}
				}
			}
			$batch .= "cd \"" . $huygens_server_image_folder . "/" . $user->name() . "/" . $image_source . "\"\n";
		}
		
		$files = $desc->files();
		
		// Preprocess the files to check for lif files with subimages. If we
		// find some, we have to:
		// (1) remove the (subImage name);
		// (2) make sure to copy the files only once.
		$filteredFiles = array( );
		$counter = -1;
		foreach ($files as $file) {
			$counter++;
			$match = array( );
			if ( preg_match("/^(.*\.lif)\s\((.*)\)/i", $file, $match) ) {
				$filteredFiles[ $counter ] = $match[ 1 ];
			} else {
				$filteredFiles[ $counter ] = $file;
			}
		}
		$files = array_unique( $filteredFiles );
		
		// Now copy the files
		foreach ($files as $file) {
			$path = split("/", $file);
			if (sizeof($path) > 0) {
				for ($i = 0; $i < sizeof($path) - 1; $i++) {
					$batch .= "-mkdir \"" . $path[$i] . "\"\n";
					$batch .= "cd \"" . $path[$i] . "\"\n";
				}
			}
			$filename = $image_folder . "/" . $user->name() . "/" . $image_source . "/" . $file;
			if (stristr($filename, ".ics"))  {
				$batch .= "put \"" . $filename . "\"\n";
				$filename = substr($filename, 0, strrpos($filename, '.ics')) . ".ids";
				$batch .= "put \"" . $filename . "\"\n";
			}
			else if (stristr($filename, ".tif") || stristr($filename, ".tiff")) {
				// TODO: if ImageFileFormat = single TIFF file, do not send corresponding series
				$basename = preg_replace("/([^_]+|\/)(_)(T|t|Z|z|CH|ch)([0-9]+)(\w+)(\.)(\w+)/", "$1$6$7", $filename);
				$name = preg_replace("/(.*)\.tiff?$/", "$1", $basename);
				$batch .= "put \"" . $name . "*\"\n";
			}
			else if (stristr($filename, ".stk")) {
				// if ImageFileFormat = STK time series, send all timepoints
				if (stripos($filename, "_t")) {
					$basename = preg_replace("/([^_]+|\/)(_)(T|t)([0-9]+)(\.)(\w+)/", "$1", $filename);
					//$name = preg_replace("/(.*)\.[STK|stk]$/", "$1", $basename);
					$batch .= "put \"" . $basename . "*\"\n";
				}
				else {
					$batch .= "put \"" . $filename . "\"\n";
				}
			}
			else {
				$batch .= "put \"" . $filename . "\"\n";
			}
			if (sizeof($path) > 0) {
				for ($i = 0; $i < sizeof($path) - 1; $i++) {
					$batch .= "cd ..\n";
				}
			}
		}
		
		$batch .= "cd ..\n";
		$batch .= "-mkdir \"" . $image_destination . "\"\n";
		$batch .= "quit\n";
				
		// report("\nBATCH \n$batch", 2);
		
		$batch_filename = $image_folder . "/" . $user->name() . "/" . "batchfile_" . $desc->id();
		
		$batchfile = fopen($batch_filename, 'w');
		fwrite($batchfile, $batch);
		fclose($batchfile);
		
		// TODO refactor this << move to Shell
		exec("sftp -b " . $batch_filename . " " . $huygens_user . "@" . $server_hostname);
                exec("rm -f " . $batch_filename);
		// >>
		
		return $huygens_server_image_folder . $user->name() . "/" . $image_source . "/";
	}
	
	function nextJobFromQueue() {
		$queue = $this->queue;
		$foundExecutableJob = False;
		$pausedJobs = False;
		$jobDescription = $queue->getNextJobDescription();
		while ($jobDescription != NULL && !$foundExecutableJob) {
			$user = $jobDescription->owner();
			$username = $user->name();
			$fileserver = new Fileserver($username);
			if ($fileserver->isReachable()) {
				$foundExecutableJob = True;
			} else {
                                $src = $fileserver->sourceFolder();
                                $dest = $fileserver->destinationFolder();
                                report("fileserver not reachable: $src or $dest".
                                "do not exist", 2);
				$pausedJobs = True;
				$queue->pauseJob($jobDescription);
				return NULL;
			}
			$jobDescription = $queue->getNextJobDescription();
		}
		if ($pausedJobs)
			$queue->restartPausedJobs();
		if ($jobDescription == NULL)
			return NULL;
		$job = new Job($jobDescription);
		$job->setServer($this->freeServer);
		return $job;
	}


	function cleanUpFileServer($job) {
		report("cleaning up file server", 1);
		$server = $job->server();
		// server name without proc number
		$s = split(" ", $server);
		$server_hostname = $s[0];
		$desc = $job->description();
		$user = $desc->owner();
		$username = $user->name();
		$fileserver = new Fileserver($username);
		$path = $fileserver->sourceFolder();
		$desc = $job->description();
		$queue = $this->queue;
		// finished. remove job, write email, clean up huygens server
		// clean up server
		//$scriptfilename = $path . '/' . $job->scriptName();
		//if (file_exists($scriptfilename)) unlink($scriptfilename);
		//report("removing script", 1);
		$id = $desc->id();
		$finishedMarker = $path . '/' . '.finished_' . "$id";
		if (file_exists($finishedMarker)) {
			report("removing finished marker", 1);
			unlink($finishedMarker);
		}
		$endTimeMarker = $path . '/' . '.EstimatedEndTime_' . "$id";
		if (file_exists($endTimeMarker)) {
			report("removing EstimatedEndTime report", 1);
			unlink($endTimeMarker);
		}
		// remove job
		$this->stopTime = $queue->stopJob($job);
		report("stopped job (" . date("l d F Y H:i:s") . ")\n", 1);
	}
	
	// TODO refactor this
	/*function retrieveImagesFromServer($server_hostname, $username) {
		global $imageProcessingIsOnQueueManager;
		global $image_folder;
		global $image_source;
		global $image_destination;
		global $huygens_server_image_folder;
		global $huygens_user;
		global $huygens_group;
		global $copy_images_to_huygens_server;
		if (!$imageProcessingIsOnQueueManager && $copy_images_to_huygens_server) {
			$lpath = $image_folder . "/" . $username;
			$rpath = $huygens_server_image_folder . $username;
			error_log("retrieving files .................");
			//error_log("("."scp -r ".$huygens_user."@".$server_hostname.":".$rpath."/".$image_destination." ".$lpath.")");
			$output = exec("scp -r " . $huygens_user . "@" . $server_hostname . ":" . $rpath . "/" . $image_destination . " " . $lpath);
			error_log("> " . $output);
			error_log("deleting files ...................\n");
			exec("ssh " . $huygens_user . "@" . $server_hostname . " rm -f " . $rpath . "/" . $image_source . "/" . $job->scriptName());
			//error_log("("."ssh ".$huygens_user."@".$server_hostname." rm -f ".$rpath."/".$image_source."/".$job->scriptName().")");
			exec("ssh " . $huygens_user . "@" . $server_hostname . " rm -rf " . $rpath . "/" . $image_destination . "/*");
			exec("ssh " . $huygens_user . "@" . $server_hostname . " rm -rf " . $rpath . "/" . $image_source . "/*");
			//error_log("("."ssh ".$huygens_user."@".$server_hostname." rm -rf ".$rpath."/".$image_destination."/*".")");
			error_log("> " . $output);
			exec("chown -R " . $huygens_user . ":" . $huygens_group . " " . $lpath);
		}
	}*/
        
        // TODO refactor semantics of this method
        function restoreOwnership($username) {
                global $resultImagesOwnedByUser;
                global $image_user;
                global $image_group;
                global $image_folder;
                
                //if ($resultImagesOwnedByUser)
                //      ;
                //else
                        //$result = exec("chown -R " . $huygens_user . ":" . $huygens_group . " " . $image_folder . "/" . $username);
                        //error_log($result);
                        
                $result = exec("sudo chown -R " . $image_user . ":" . $image_group . " " . $image_folder . "/" . $username);
                error_log("Restoring ownership... " . $result);
        }
	
	// TODO refactor this
	function updateJobAndServerStatus() {
		global $imageProcessingIsOnQueueManager;
		global $use_accounting_system;
		global $accounting_client_name;
		global $send_mail;
		global $logdir;
		
		// TODO check if it is necessary
		$queue = $this->queue;
                // Kill marked running jobs
		$queue->killMarkedJobs();
		// Remove broken jobs
		if ($queue->removeMarkedJobs()) {
                        report("broken jobs removed", 2);
		}
		$runningJobs = $queue->runningJobs();
        if (count($runningJobs) > 0) {
            report(count($runningJobs) . " job" . (count($runningJobs) == 1 ? " is" : "s are") . " running", 2);
            // Because something is running, we are not in a hurry to continue.
            // Delay execution.
            sleep(5);
        }
		foreach ($runningJobs as $job) {
			$desc = $job->description();
			$user = $desc->owner();
			$id = $job->id();
			
			// Check if fileserver is reachable
			$fileserver = new Fileserver($user->name());
			if (!$fileserver->isReachable())
				continue;
			
			// Check if Huygens host is reachable
			$proc = newExternalProcessFor($job->server(),
                                $job->server() . "_" .$job->id() . "_out.txt",
                                $job->server() . "_" .$job->id(). "_error.txt");
			if (!$proc->ping())
				continue;
			
			// Check finished marker
			$finished = $job->checkProcessFinished();
			if (!$finished) {
                            continue;
                        }
			report("checked finished process", 2);
			
			// Check result image
			$resultSaved = $job->checkResultImage();
			report("checked result image", 2);
			
			// If fileshare is not on the same host as Huygens
			if (!$imageProcessingIsOnQueueManager)
				$this->restoreOwnership($user->name());
			
			// Check if error occured
			$errorOccured = $this->errorOccured($job);
			
			// Notify user
			$startTime = $queue->startTime($job);
                        $errorFile = $logdir . "/" . $job->server() .
                            "_" .$job->id(). "_error.txt";
                        $logFile = $logdir . "/" . $job->server() .
                            "_" .$job->id(). "_out.txt";
			// TODO add warning messages notification
			if (!$resultSaved || $errorOccured) {
				report("finishing job " . $desc->id() . " with error on " . $job->server(), 1);
				// Clean up server
				$this->cleanUpFileServer($job);
				// Reset server and remove job from the job queue (update database) 
				$this->stopTime = $queue->stopJob($job);
				// Write email
				$message = "";
                                if (!$resultSaved) {
                                    $message .= "\nNo result file stored in the destination directory.\n";
                                }
				if (file_exists($errorFile))
                                    $message .= "\n\n- HUYGENS ERROR REPORT (stderr) --------------\n\n". file_get_contents($errorFile);
                                if (file_exists($logFile))
                                    $message .= "\n\n- HUYGENS REPORT (stdout) --------------------\n\n". file_get_contents($logFile);
				if ($send_mail) 
                                    $this->notifyError($job, $message, $startTime);

                                if (file_exists($errorFile)) {
                                    $gFile = $logdir . "/" . $job->server() .
                                    "_error.txt";
                                    # Accumulate error logs
                                    exec ("cat \"$errorFile\" >> \"$gFile\"");
                                    unlink($errorFile);	
                                }

				if (file_exists($logFile))
					unlink($logFile);	
			}
			else {
				report("job " . $desc->id() . 
                                        " completed on " . $job->server(), 1);
				// Report information to statistics table
				$db = new DatabaseConnection();
				$db->updateStatistics($job, $startTime);
				// Clean up server
				$this->cleanUpFileServer($job);
				// Reset server and remove job from the job queue (update database) 
				$this->stopTime = $queue->stopJob($job);
				$this->writeParameterFile($job, $startTime,
                                        $logFile, $errorFile);
				// Write email
				if ($send_mail) $this->notifySuccess($job, $startTime);
				if (file_exists($errorFile)) { 
				    unlink($errorFile);
				} 
				if (file_exists($logFile))
					unlink($logFile);	
				// make withdrawal of hours
				if ($use_accounting_system) $this->makeWithdrawal($desc->owner(), $accounting_client_name, $startTime, $this->stopTime, $desc->credit(), $desc->group);
                                # $proc = $this->activeProcs[$id];
                                # $debug = var_export($this->activeProcs, true);
                                # report ("Releasing shell, ".
                                 #       "unsetting activeProc $id", 2);
                                # $proc->release();
                                # unset($this->activeProcs[$id]);
			}
			
			/*$result = False;
			if ($imageProcessingIsOnQueueManager || (!$imageProcessingIsOnQueueManager && $finished)) {
				$result = $job->checkResultImage();
				report("checked result image", 2);
			}*/
			
			// TODO check for redundant file retrieval
			/*if (!$imageProcessingIsOnQueueManager && $result) {
				$this->retrieveImagesFromServer($server_hostname, $username);
			}*/
			
			// Notify user
			/*$startTime = $queue->startTime($job);
			$errorOccured = $this->errorOccured($job);
			// TODO
			$warningOccured = file_exists($job->server() . '_warning.txt');
			// manage warning messages when processing a job
			// TODO refactor
			if ($finished && $warningOccured) {
				$warnings = file_get_contents($job->server() . '_warning.txt');
				$input = $warnings;
				if ($warnings)
					$input = ereg_replace("^.*\[$id] ([a-zA-Z ]*)\n.*$", "\\1", $warnings);
				if ($input != $warnings) {
					report("finishing job $id with warning on " . $job->server(), 1);
					// warning
					$this->cleanUpFileServer($job);
					// remove job
					$this->stopTime = $queue->stopJob($job);
					// write email
					$message = $input;
					if ($send_mail) $this->notifyWarning($job, $message, $startTime);
					//if (file_exists($errorFile)) { 
					//    unlink($errorFile);
					//}
				}
			} else
				if (($finished && !$result) || $errorOccured) {
					report("finishing job $id with error on " . $job->server(), 1);
					// error
					$this->cleanUpFileServer($job);
					// remove job
					$this->stopTime = $queue->stopJob($job);
					// write email
					$message = '';
					if (file_exists($errorFile)) {
						$message = file_get_contents($errorFile);
					}
					if ($send_mail) $this->notifyError($job, $message, $startTime);
					//if (file_exists($errorFile)) { 
					//    unlink($errorFile);
					//} 
				}
				else if ($finished && $result) {
					report("job $id completed on " . $job->server(), 1);
					// finsihed. remove job, write email, clean up huygens server
					// clean up server
					$this->cleanUpFileServer($job);
					// remove job
					$this->stopTime = $queue->stopJob($job);
					// write email
					$this->writeParameterFile($job, $startTime, $logFile, $errorFile);
					if ($send_mail) $this->notifySuccess($job, $startTime);
					//if (file_exists($errorFile)) { 
					//    unlink($errorFile);
					//} 
					// make withdrawal of hours
					if ($use_accounting_system) {
						$this->makeWithdrawal($desc->owner(), $accounting_client_name, $startTime, $this->stopTime, $desc->credit(), $desc->group);
					}
				}*/
		}
	}

	function makeWithdrawal($user, $client, $startTime, $stopTime, $credit, $group) {
		global $accounting_transaction_path;
		$stopTimeForFilename = str_replace(" ", "-", $stopTime);
		$filename = $accounting_transaction_path . str_replace(":", "-", $stopTimeForFilename);
		$startTimeForMessage = str_replace(":", "/", $startTime) . ".0";
		$endTimeForMessage = explode(".", $stopTime);
		$endTimeForMessage = str_replace(":", "/", $endTimeForMessage[0]) . ".0";
		$message = 'MessageRequestExecuteWithdrawal:' . $user->name() . ':' . $client . ':' . $startTimeForMessage . ':' . $endTimeForMessage . ':' . $credit . ":" . $group;
		$file = fopen($filename, 'w');
		fwrite($file, $message);
		fclose($file);
		exec("hrmAccountingRelay");
		report("created transaction: $message", 1);
	}

	function writeParameterFile($job, $startTime, $logFile, $errorFile) {
		global $resultImagesOwnedByUser;
                global $imageProcessingIsOnQueueManager;
		$result = False;
		$desc = $job->description();
		$sourceFileName = $desc->sourceImageNameWithoutPath();
		$destFileName = $desc->destinationImageNameWithoutPath();
		$text = '';
		$text = $text . "Your job started at $startTime and finished".
                    " at " . date("Y-m-d H:i:s") . ".\n\n";
		$text = $text . "The image $sourceFileName has been ".
                    "successfully treated by Huygens.\n\n";
                $text = $text . "You will find the resulting image ".
                    "($destFileName) in your destination folder.\n";
		$text = $text . $this->parameterText($job);
		$imageName = $desc->destinationImageName();
		$user = $desc->owner();
		$username = $user->name();
		$fileserver = new Fileserver($username);
		$path = $fileserver->destinationFolderFor($desc);

                if (file_exists($errorFile))
                    $text .= "\n\n- HUYGENS ERROR REPORT (stderr) --------------\n\n". file_get_contents($errorFile);
                if (file_exists($logFile))
                    $text .= "\n\n- HUYGENS REPORT (stdout) --------------------\n\n". file_get_contents($logFile);

                $parameterFileName = $path . $imageName . '.log.txt';

                // Why (over)writing this file only if it already exists? Make
                // it true by now to always write it:
		if (True || file_exists($parameterFileName)) {
			//$parameterFileName = $path . '/' . $imageName . '.txt';
			$file = fopen($parameterFileName, "w");
			$result = !$result && (fwrite($file, $text) > 0);
			fclose($file);
			if ($resultImagesOwnedByUser) {
				chown($parameterFileName, $username);
			}
			// TODO refactor this
			else if (!$imageProcessingIsOnQueueManager) {
				$this->restoreOwnership($user->name());
			}
		}
		return $result;
	}
	
	// TODO check what happens if a job fails and the next one succeeds
	// TODO refactor this!
	function errorOccured($job) {
		global $imageProcessingIsOnQueueManager;
		//if ($imageProcessingIsOnQueueManager) {
			return False;
		//}
		// The following is never run, but for completeness we add the logdir
		global $logdir;
		$errorFile = $logdir . "/" . $job->server() .  "_" .$job->id(). '_error.txt';
		$message = '';
		if (file_exists($errorFile)) {
			$message = file_get_contents($errorFile);
		}
		if (strstr($message, 'SignalError')) {
			return True;
		}
		if (strstr($message, 'failed to allocate')) {
			return True;
		}
		if (strstr($message, 'Error encountered')) {
			return True;
		}
		return False;
	}

	function notifySuccess($job, $startTime) {
		global $email_sender;
                global $hrm_url;
                global $useThumbnails;
		$desc = $job->description();
		$user = $desc->owner();
                $fileserver = new Fileserver($user->name());
		$emailAddress = $user->emailAddress();
		$sourceFileName = $desc->sourceImageNameWithoutPath();
		//$destFileName = $desc->destinationImageName();
		$destFileName = $desc->destinationImageNameWithoutPath();
		$text = "This is a mail generated automatically by the Huygens Remote Manager.\n";
		$text = $text . "Your job started at $startTime and finished at " . date("Y-m-d H:i:s") . ".\n";
		$text = $text . "The image $sourceFileName was successfully processed by Huygens.\n";
		$text = $text . "You will find the resulting image ($destFileName) in your destination folder.\n";
                $imageName = $desc->destinationImageNameAndExtension();
                if ( $useThumbnails ) {
                    $link = $hrm_url.
                           "/file_management.php?compareResult=".
                           urlencode($imageName);
                    $text .= "\nYou can find a preview of the result at ".$link.
                             " (login required).\n\n";
                }
                $remarksFile =
                    $fileserver->destinationFolder(). "/".
                    $imageName.".remarks.txt";
		if (file_exists($remarksFile)) {
                    $text = $text . file_get_contents($remarksFile)."\n\n";
                }
		$text = $text . "Best regards,\n";
		$text = $text . "Huygens Remote Manager\n";
		$text = $text . $this->parameterText($job);
		$mail = new Mail($email_sender);
		$mail->setReceiver($emailAddress);
		$mail->setSubject('Your huygens job finished successfully');
		$mail->setMessage($text);
		$mail->send();
	}

	function notifyError($job, $message, $startTime) {
		global $email_sender;
		$job->createScript();
		$desc = $job->description();
		$user = $desc->owner();
		$emailAddress = $user->emailAddress();
		$sourceFileName = $desc->sourceImageNameWithoutPath();
		//$destFileName = $desc->destinationImageName();
		$text = "\nThis is a mail generated automatically by the Huygens Remote Manager.\n\n";
		$text = $text . "Sorry, the processing of the image \n$sourceFileName\nhas been terminated with an error.\n\n";
		$text = $text . "Your job started at $startTime and failed at " . date("Y-m-d H:i:s") . ".";
		$text = $text . "\n\n";
		$text = $text . "Best regards,\n";
		$text = $text . "Huygens Remote Manager\n";
		$text = $text . "-------------------------------------";
		$text = $text . "\n\n";
		$text = $text . $message;
		$text = $text . "\n\n- SUMMARY ---------------------------\n\n";
		$text = $text . $this->parameterText($job);

		$text = $text . "\n\n- SCRIPT ----------------------------\n\n";
		$text = $text . "What follows is the Huygens Core script executed when the error occured:\n\n";
		$text = $text . $job->script();
		$text = $text . "\n\n-------------------------------------\n\n";
		$mail = new Mail($email_sender);
		$mail->setReceiver($emailAddress);
		$mail->setSubject('Your huygens job finished with an error');
		$mail->setMessage($text);
		$mail->send();
		// also notify error to admin
		$db = new DatabaseConnection();
		$mail->setReceiver($db->emailAddress('admin'));
		$mail->send();
	}

	// notify the user of interactive dialogs silently managed by Huygens
	function notifyWarning($job, $message, $startTime) {
		global $email_sender;
		$job->createScript();
		$desc = $job->description();
		$user = $desc->owner();
		$emailAddress = $user->emailAddress();
		$sourceFileName = $desc->sourceImageNameWithoutPath();
		//$destFileName = $desc->destinationImageName();
		$text = "This is a mail generated automatically by the Huygens Remote Manager.\n";
		$text = $text . "The processing of the image \n$sourceFileName\n was completed with a warning: ";
		$text = $text . $message . "\n";
		$text = $text . "Huygens used a theoretical PSF instead of the measured PSF you provided.\n";
		$text = $text . "Your job started at $startTime and finished at " . date("Y-m-d H:i:s") . ".";
		$text = $text . "\n";
		$text = $text . "Best regards,\n";
		$text = $text . "Huygens Remote Manager\n";
		$text = $text . $this->parameterText($job);
		$mail = new Mail($email_sender);
		$mail->setReceiver($emailAddress);
		$mail->setSubject('Your huygens job finished with a warning');
		$mail->setMessage($text);
		$mail->send();
		// also notify error to admin
		/*$db = new DatabaseConnection();
		$mail->setReceiver($db->emailAddress('admin'));
		$mail->send();*/
	}
	
	function notifyPingError($name) {
                report("A ping error notification was sent on " . date("r", time()), 2);
                global $email_sender;
                global $email_admin;
                $text = "Huygens Remote Manager warning:\n"
                        . $name . " could not be pinged on " . date("r", time());
                $mail = new Mail($email_sender);
		$mail->setReceiver($email_admin);
		$mail->setSubject('Huygens Remote Manager - ping error');
		$mail->setMessage($text);
		$mail->send(); 
        }

	function parameterText($job) {
		$desc = $job->description();
		$id = $desc->id();
		$pid = $job->pid();
		$server = $job->server();
		$result = '';
		$result = $result . "\nJob id: $id (pid $pid on $server)\n";
		$result = $result . "\nImage parameters:\n\n";
		$parameterSetting = $desc->parameterSetting();
		$parameterSettingString = $parameterSetting->displayString();
		$result = $result . $parameterSettingString;
		$result = $result . "\nRestoration parameters:\n\n";
		$taskSetting = $desc->taskSetting();
		$taskSettingString = $taskSetting->displayString();
		$result = $result . $taskSettingString;
		return $result;
	}

	function getFreeServer() {
		$db = new DatabaseConnection();
		$serverNames = $db->availableServer();
		foreach ($serverNames as $name) {
			$status = $db->statusOfServer($name);
			if ($status == 'free') {
				$proc = newExternalProcessFor($name, $name . "_out.txt", $name .  "_error.txt");
				//report("found free server", 2);
				if ($proc->ping()) {
					//report("ping succeeded for $name", 2);
                                        $this->nping[$name] = 0;
					$this->freeServer = $name;
					return True;
				}
                                else {
                                        $this->incNPing($name);
                                        if ($this->nping[$name]  ==  40) {
                                                $this->notifyPingError($name);
                                        }
                                }
			}
		}
		$this->freeServer = False;
		return $this->freeServer;
	}
	
	function incNPing($name) {
		$this->nping[$name] ++;
	}

	function stop() {
		$this->shallStop = True;
	}

	function shallStop() {
		if ($this->shallStop)
			return True;
		$this->waitForDatabaseConnection();
		$db = new DatabaseConnection();
		$this->shallStop = !$db->isSwitchOn();
		return $this->shallStop;
	}

	function waitForDatabaseConnection() {
		$isDatabaseReachable = False;
		while (!$isDatabaseReachable) {
			$db = new DatabaseConnection();
			if ($db->isReachable()) {
				$isDatabaseReachable = True;
			}
		}
	}
	
	function readHuCoreVersion() {
		$huversion = askHuCore("reportVersionNumberAsInteger");	
		$huversion = $huversion["version"];	
		report("HuCore version = " . $huversion . "\n", 2);	
		$db = new DatabaseConnection();
		if(!$db->setHuCoreVersion($huversion))
			return false;
		return true;
	}

        function run() {
            global $imageProcessingIsOnQueueManager;
            global $logdir;

            $queue = $this->queue;

            $this->waitForDatabaseConnection();
            $server = $queue->availableServer();
            // TODO refactor this in order to manage error logs per job
            foreach ($server as $name) {
                $this->nping[$name] = 0;
            }

            report("Huygens Remote Manager started on " 
                    . date("Y-m-d H:i:s") . "\n", 1);
	    
	    if(!$this->readHuCoreVersion()) {
		error_log("An error occurred while reading HuCore version");
		return;
	    }

            while (!$this->shallStop()) {
                set_time_limit(0);
                $this->queue = $queue;
                $result = True;

                // Reduce the used cycles by going to sleep for one second
                if ($imageProcessingIsOnQueueManager) {
                    sleep(1);
                }

                // Check if jobs finished and update the database. Inform the
                // user via email.
                $this->updateJobAndServerStatus();
                //report("Huygens server updated status", 2);

                // Read in a free huygens server
                while (!( $queue->isLocked() ) && $this->getFreeServer() ) {

                    $job =  $this->nextJobFromQueue();
                    // Exit the loop when no job is queued.
                    if ( $job == NULL ) { break; }

                    report("using Huygens server: " . $this->freeServer, 2);

                    // Read in a queued job 
                    $desc = $job->description();
                    $id = $desc->id();
                    report("processing job " . $id ." on " . $job->server(), 1);

                    // TODO check this <<
                    // If the job is compound create sub jobs and
                    // remove job otherwise create script
                    //$result = $result && $job->createSubJobsOrScript();
                    $result = $job->createSubJobsOrScript();
                    if (!$result || $desc->isCompound()) {
                        error_log("error or compound job");
                        continue;
                    }
                    report("script has been created", 1);
                    // Execute the script on the Huygens server and
                    // update the database state
                    $result = $result && $this->executeScript($job);
                    if (!$result)
                        continue;
                    report("script has been executed", 1);
                    $result = $result && $queue->startJob($job);
                    report("job has been started (" 
                            . date("Y-m-d H:i:s") . ")", 1);
                }
            }
            report("Huygens Remote Manager stopped via database switch on " 
                    . date("Y-m-d H:i:s"), 1);
        }
}
?>
Return current item: Huygens Remote Manager