Location: PHPKode > scripts > Back Propagation Scale > class.BackPropagationScale.php
<?php
/**
 * file: class.BackPropagationScale.php
 *
 * Using artificial intelligence and NN (neuronal networks)
 * to solve the multiplication table problem.
 *
 * It uses a technique called back propagation, that is
 * the network learns by calculating the errors going
 * backwards: from the output, through the first hidden layers.
 *
 * The learning unit is called the perceptron: a neuron capable
 * of learning, connected to layers and adjusting the weights.
 *
 * Data needs to be scaled/unscaled to adjust the values common to
 * the neural network
 *
 * Free for educational purposes
 * Copyright © 2010
 *
 * @author freedelta ( http://freedelta.free.fr ) 
 */
error_reporting(E_ERROR);
define("_RAND_MAX",32767);
define("HI",0.9);
define("LO",0.1);

class BackPropagationScale
{
/* Output of each neuron */
public $output=null;

/* Last calcualted output value */
public $vectorOutput=null;

/* delta error value for each neuron */
public $delta=null;

/* Array of weights for each neuron */
public $weight=null;

/* Num of layers in the net, including input layer */
public $numLayers=null;

/* Array num elments containing size for each layer */
public $layersSize=null;

/* Learning rate */
public $beta=null;

/* Momentum */
public $alpha=null;

/* Storage for weight-change made in previous epoch (three-dimensional array) */
public $prevDwt=null;

/* Data */
public $data=null;

/* Test Data */
public $testData=null;

/* N lines of Data */
public $NumPattern=null;

/* N columns in Data */
public $NumInput=null;

/* Minimum value in data set */
public $minX=0;

/* Maximum value in data set */
public $maxX=1;

/* Stores ann scale calculated parameters */
public $normalizeMax=null;
public $normalizeMin=null;

/* Holds all output data in one array */
public $output_vector=null;

public function __construct($numLayers,$layersSize,$beta,$alpha,$minX,$maxX)
{		
    
    $this->alpha=$alpha;
    $this->beta=$beta;
    $this->minX=$minX;
    $this->maxX=$maxX;
    
    //	Set no of layers and their sizes
    $this->numLayers=$numLayers;
    $this->layersSize=$layersSize;
    
    //	seed and assign random weights
    for($i=1;$i<$this->numLayers;$i++)
    {
        for($j=0;$j<$this->layersSize[$i];$j++)
        {
            for($k=0;$k<$this->layersSize[$i-1]+1;$k++)
            {
                $this->weight[$i][$j][$k]=$this->rando();
            }
            // bias in the last neuron				
            $this->weight[$i][$j][$this->layersSize[$i-1]]=-1;
        }
    }	
    
    //	initialize previous weights to 0 for first iteration
    for($i=1;$i<$this->numLayers;$i++)
    {
        for($j=0;$j<$this->layersSize[$i];$j++)
        {
            for($k=0;$k<$this->layersSize[$i-1]+1;$k++)
            {					
                $this->prevDwt[$i][$j][$k]=(double)0.0;
            }				
        }
    }
}

public function rando()
{
    $randValue = LO + (HI - LO) * mt_rand(0, _RAND_MAX)/_RAND_MAX;  
    return $randValue;//32767   
}

/* ---	sigmoid function */
public function sigmoid($inputSource)
{
    return (double)(1.0 / (1.0 + exp(-$inputSource)));	
}

/* --- mean square error */
public function mse($target)
{	
    $mse=0;
    
    for($i=0;$i<$this->layersSize[$this->numLayers-1];$i++)
    {
        $mse+=($target-$this->output[$this->numLayers-1][$i])*($target-$this->output[$this->numLayers-1][$i]);		
    }
    return $mse/2;
}

/* ---	returns i'th outputput of the net */
public function Out($i)
{
    return $this->output[$this->numLayers-1][$i];
}
/* ---
 * Feed forward one set of input
 * to update the output values for each neuron. This function takes the input 
 * to the net and finds the output of each neuron
 */
public function ffwd($inputSource)
{	
    $sum=0.0;
    
    $numElem=count($inputSource);
    //	assign content to input layer
     for($i=0;$i<$numElem;$i++)
    {
        $this->output[0][$i]=$inputSource[$i];  // outputput_from_neuron(i,j) Jth neuron in Ith Layer		
    }
        
    //	assign output (activation) value to each neuron usng sigmoid func
    for($i=1;$i<$this->numLayers;$i++)										// For each layer
    {	
        for($j=0;$j<$this->layersSize[$i];$j++)								// For each neuron in current layer
        {	
            $sum=0.0;
            for($k=0;$k<$this->layersSize[$i-1];$k++)						// For each input from each neuron in preceeding layer
            {	
                $sum+=$this->output[$i-1][$k]*$this->weight[$i][$j][$k];	// Apply weight to inputs and add to sum	
            }
            // Apply bias
            $sum+=$this->weight[$i][$j][$this->layersSize[$i-1]];	
            // Apply sigmoid function					
            $this->output[$i][$j]=$this->sigmoid($sum);						
        }
    }	
}

/* ---	Backpropagate errors from outputput	layer back till the first hidden layer */
public function bpgt($inputSource,$target)
{	
    /* ---	Update the output values for each neuron */
    $this->ffwd($inputSource);

    ///////////////////////////////////////////////
    /// FIND DELTA FOR OUPUT LAYER (Last Layer) ///
    ///////////////////////////////////////////////
    
    for($i=0;$i<$this->layersSize[$this->numLayers-1];$i++)
    {	
        $this->delta[$this->numLayers-1][$i]=$this->output[$this->numLayers-1][$i]*(1-$this->output[$this->numLayers-1][$i])*($target-$this->output[$this->numLayers-1][$i]);
    }
    
    /////////////////////////////////////////////////////////////////////////////////////////////
    /// FIND DELTA FOR HIDDEN LAYERS (From Last Hidden Layer BACKWARDS To First Hidden Layer) ///
    /////////////////////////////////////////////////////////////////////////////////////////////
    
    for($i=$this->numLayers-2;$i>0;$i--)
    {
        for($j=0;$j<$this->layersSize[$i];$j++)
        {
            $sum=0.0;
            for($k=0;$k<$this->layersSize[$i+1];$k++)
            {
                $sum+=$this->delta[$i+1][$k]*$this->weight[$i+1][$k][$j];
            }			
            $this->delta[$i][$j]=$this->output[$i][$j]*(1-$this->output[$i][$j])*$sum;
        }
    }
    
    ////////////////////////
    /// MOMENTUM (Alpha) ///
    ////////////////////////
    
    for($i=1;$i<$this->numLayers;$i++)
    {
        for($j=0;$j<$this->layersSize[$i];$j++)
        {
            for($k=0;$k<$this->layersSize[$i-1];$k++)
            {
                $this->weight[$i][$j][$k]+=$this->alpha*$this->prevDwt[$i][$j][$k];				
            }
            $this->weight[$i][$j][$this->layersSize[$i-1]]+=$this->alpha*$this->prevDwt[$i][$j][$this->layersSize[$i-1]];
        }
    }
    
    ///////////////////////////////////////////////
    /// ADJUST WEIGHTS (Using Steepest Descent) ///
    ///////////////////////////////////////////////
    
    for($i=1;$i<$this->numLayers;$i++)
    {
        for($j=0;$j<$this->layersSize[$i];$j++)
        {
            for($k=0;$k<$this->layersSize[$i-1];$k++)
            {
                $this->prevDwt[$i][$j][$k]=$this->beta*$this->delta[$i][$j]*$this->output[$i-1][$k];
                $this->weight[$i][$j][$k]+=$this->prevDwt[$i][$j][$k];
            }
            /* --- Apply the corrections */
            $this->prevDwt[$i][$j][$this->layersSize[$i-1]]=$this->beta*$this->delta[$i][$j];
            $this->weight[$i][$j][$this->layersSize[$i-1]]+=$this->prevDwt[$i][$j][$this->layersSize[$i-1]];
        }
    }
}

///////////////////////////////
/// SCALING FUNCTIONS BLOCK ///
///////////////////////////////

/* --- Set scaling parameters */
public function setScaleOutput($data)
{
    $oldMin=$data[0][0];
    $oldMax=$oldMin;	
    $numElem=count($data[0]);
    
    /* --- First calcualte minimum and maximum */
    for($i=0;$i<$this->NumPattern;$i++)
    {
        $oldMin=$data[$i][0];
        $oldMax=$oldMin;	
        
        for($j=1;$j<$numElem;$j++)
        {
            // Min
            if($oldMin > $data[$i][$j])
            {
                $oldMin=$data[$i][$j];
            }
            // Max
            if($oldMax < $data[$i][$j])
            {
                $oldMax=$data[$i][$j];
            }
        }
        $this->normalizeMin[$i]=$oldMin;
        $this->normalizeMax[$i]=$oldMax;
    }
    
}

/* --- Scale input data to range before feeding it to the network */
/*
                     x - Min
    t = (HI -LO) * (---------) + LO
                     Max-Min 
*/
public function scale($data)
{
    $this->setScaleOutput($data);
    $numElem=count($data[0]);
    $temp=0.0;
    
    for( $i=0; $i < $this->NumPattern; $i++ )
    {
        for($j=0;$j<$numElem;$j++)
        {
            $temp=(HI-LO)*(($data[$i][$j] - $this->normalizeMin[$i]) / ($this->normalizeMax[$i] - $this->normalizeMin[$i])) + LO;
            $data[$i][$j]=$temp;
        }
	}
    
    return $data;
}

/* --- Unscale output data to original range */
/*
                       x - LO 
    t = (Max-Min) * (---------) + Min
                       HI-LO
*/
public function unscaleOutput($output_vector)
{	
    $temp=0.0;

    for( $i=0; $i < $this->NumPattern; $i++ )
    {
       
        $temp=($this->normalizeMax[$i]-$this->normalizeMin[$i]) * (($output_vector[$i] - LO) / (HI-LO)) + $this->normalizeMin[$i] ;	
		$unscaledVector[$i] =$temp;        
    }
    
    return $unscaledVector;
}

public function Run($dataX,$testDataX)
{
    /* --- Threshhold - thresh (value of target mse, training stops once it is achieved) */
    $Thresh =  0.00001;
    $numEpoch = 200000;	
    $MSE=0.0;	
    $this->NumPattern=count($dataX);	
    $this->NumInput=count($dataX[0]);	
    
    /* --- Pre-process data: Scale input and test values */
    $data=$this->scale($dataX);
  
    /* --- Test data=(data-1 column) */
    for($i=0;$i<$this->NumPattern;$i++)
    {
        for($j=0;$j<$this->NumInput-1;$j++)
        {
            $testData[$i][$j]=$data[$i][$j];
        }		
    }
   
    /* --- Start training: looping through epochs and exit when MSE error < Threshold */
    echo  "\nNow training the network....";	
    
    for($e=0;$e<$numEpoch;$e++)
    {
        /* -- Backpropagate */
        $this->bpgt($data[$e%$this->NumPattern],$data[$e%$this->NumPattern][$this->NumInput-1]);
                
        $MSE=$this->mse($data[$e%$this->NumPattern][$this->NumInput-1]);
        if($e==0)
        {
            echo "\nFirst epoch Mean Square Error: $MSE";
        }
        
        if( $MSE < $Thresh)		
        {
            echo "\nNetwork Trained. Threshold value achieved in ".$e." iterations.";
            echo "\nMSE:  ".$MSE;
            break;
        }
    }
    
    echo "\nLast epoch Mean Square Error: $MSE";
    
    echo "\nNow using the trained network to make predictions on test data....\n";	
    
    for ($i = 0 ; $i < $this->NumPattern; $i++ )
    {
        $this->ffwd($testData[$i]);
        $this->vectorOutput[]=(double)$this->Out(0);
    }
    
    $out=$this->unscaleOutput($this->vectorOutput);
    
    for($col=1;$col<$this->NumInput;$col++)
    {
        echo "Input$col\t";
    }
        echo "Predicted \n";
    
        for ($i = 0 ; $i < $this->NumPattern; $i++ )
    {
        for($j=0;$j<$this->NumInput-1;$j++)
        {          
            echo "  ".$testDataX[$i][$j]."  \t\t";
        }	
        echo "  " .abs($out[$i])."\n";
    }
}

}

/* --- Sample use */
// Mutliplication data: 1 x 1 = 1, 1 x 2 = 2,.. etc
$data=array(0=>array(1,1,1),
1=>array(1,2,2),
2=>array(1,3,3),
3=>array(1,4,4),
4=>array(1,5,5),
5=>array(2,1,2),
6=>array(2,2,4),
7=>array(2,3,6),
8=>array(2,4,8),
9=>array(2,5,10),
10=>array(3,1,3),
11=>array(3,2,6),
12=>array(3,3,9),
13=>array(3,4,12),
14=>array(3,5,15),
15=>array(4,1,4),
16=>array(4,2,8),
17=>array(4,3,12),
18=>array(4,4,16),
19=>array(4,5,20),
20=>array(5,1,5),
21=>array(5,2,10),
22=>array(5,3,15),
23=>array(5,4,20),
24=>array(5,5,25)

);

// 1 x 1 =?
$testData=array(0=>array(1,1),
1=>array(1,2),
2=>array(1,3),
3=>array(1,4),
4=>array(1,5),
5=>array(2,1),
6=>array(2,2),
7=>array(2,3),
8=>array(2,4),
9=>array(2,5),
10=>array(3,1),
11=>array(3,2),
12=>array(3,3),
13=>array(3,4),
14=>array(3,5),
15=>array(4,1),
16=>array(4,2),
17=>array(4,3),
18=>array(4,4),
19=>array(4,5),
20=>array(5,1),
21=>array(5,2),
22=>array(5,3),
23=>array(5,4),
24=>array(5,5)
);


$layersSize=array(3,2,1);
$numLayers = count($layersSize);

// Learing rate - beta
// momentum - alpha
$beta = 0.3;
$alpha = 0.1;

$minX=1;
$maxX=25;

// Creating the net    
$bp=new BackPropagationScale($numLayers,$layersSize,$beta,$alpha,$minX,$maxX);
$bp->Run($data,$testData);
?>
Return current item: Back Propagation Scale