Location: PHPKode > scripts > Guaranix Full Text > guaranix-full-text/tokenizer.php
<?
class structTok
{
	var $left;
	var $right;
	var $data;
	
	function structTok($data = NULL)
	{
		$this->left = $this->right = NULL;
		$this->data = $data;
	}
}

class Tokenizer
{
	var $Root;
	var $string;
	var $actual;
	var $max;
	
	function Tokenizer()
	{
		global $boolean;
		$this->Root = new structTok();
		$boolean= array('AND','OR','NOT','XOR','NEAR');
	}
	
	function parser($string)
	{
		global $boolean;
		
		$this->actual = 0;
		$this->max = strlen($string);
		$this->string = $string;
		$left = $right = $tok;
		$first = true;	
		while (!$this->isEnd())
		{
			
			$tok = $this->getTok();
			if ($tok != "")
			{
				if ( $first || array_search($tok,$boolean) !== false || array_search($last,$boolean) !== false)
				{
					$this->addRoot($tok);
					//print $tok."<br>\r\n";
				}
				else
				{
					$this->addRoot("AND");
					$this->addRight($tok);
					//print "AND\r\n<br>".$tok."<br>\r\n";
				}
				$first = false;
				$last = $tok;
			}
		}
		$this->Tree2Array();
	}
	
	function getTok()
	{
		$bef = $this->actual;
		$frase = false;
		$str = substr($this->string,$bef);
		
		for	(; $this->actual < $this->max; $this->actual++)
		{
			if ($this->string[$this->actual] == '"' && $frase == false)
				$frase = true;
			else if ($this->string[$this->actual] == '"' && $frase == true)
			{
				$str = substr($this->string,$bef + 1,$this->actual-$bef - 1);
				break;
			}
			if ($this->string[$this->actual] == ' ' && $frase == false)
			{
				$str = substr($this->string,$bef,$this->actual-$bef);
				break;
			}
		}
		
		$this->actual++;
		return trim($str);
	}
	
	function isEnd()
	{
		if ($this->max > $this->actual)
			return false;
		else
			return true;
	}
	
	function addLastLeft($data)
	{
		$root = &$this->Root;
		while ( $root->left != NULL )
		{
			$root = &$root->left;
		}
		$new = new structTok($data);
		$root->left = &$new;
	}
	
	function addRoot($word)
	{
		if ($this->Root != NULL)
			$lastRoot = &$this->Root;
		$Root = new structTok($word);
		$this->Root = &$Root;
		if ($lastRoot != NULL)
			$Root->left = &$lastRoot;
	}
	
	function addRight ($data)
	{
		$root = &$this->Root;
		while ( $root->right != NULL )
		{
			$root = &$root->right;
		}
		$new = new structTok($data);
		$root->right = &$new;
	}
	
	function addLastRight($data)
	{
		$root = &$this->Root;
		while ( $root->right != NULL )
		{
			$root = &$root->right;
		}
		$new = new structTok($data);
		$root->right = &$new;
	}
	
	function printree($tree = 'vacio')
	{
		if ($tree == 'vacio')
			$tree = &$this->Root;
			
		if ($tree == NULL)
			return;
			
		$this->printree(&$tree->left);
			print $tree->data."<br>";
		$this->printree(&$tree->right);	
	}
	
	function Tree2Array($tree = 'vacio')
	{
		global $TreeArray;
		if ($tree == 'vacio')
			$tree = &$this->Root;
			
		if ($tree == NULL)
			return;
			
		$this->Tree2Array(&$tree->left);
			$TreeArray[] =  $tree->data;
		$this->Tree2Array(&$tree->right);	
	}
	
	
	
}


?>
Return current item: Guaranix Full Text