Jorge Albaladejo

$spider = new spider();
 
$spider->crawl( array(  'entry'		=> 'http://jorgealbaladejo.com',
			'domains'	=> array(	'jorgealbaladejo.com' ,
							'www.jorgealbaladejo.com',
							'jorgealbaladejo.ch',
							'jorgealbaladejo.es'	) 
			) );
 
$spider->printResults();

/**
 * Class spider
 * Creates a spider which crawls the internet
 * 
 */
class spider
{
	/**
	 * @var Code Igniter object for database access
	 * 
	 */
	var $ci;
 
	/**
	 * @var maximum recursivity depth
	 * 
	 */
	var $max_depth 	= 100;
 
	/**
	 * @var default domains to restrict crawling
	 * 
	 */
	var $domains 	= array('jorgealbaladejo.com','vwww.jorgealbaladejo.com');
 
	/**
	 * @var indentation level
	 * 
	 */
	var $indent 	= 20;
 
	/**
	 * @var internal links array
	 * 
	 */
	var $links 	= array();
 
	/**
	 * Constructor
	 * 
	 */
	public function spider()
	{
		$this->ci =& get_instance();	
 
		// load links in database to internal list
		$this->_loadLinks();	
	}
 
	/**
	 * Entry point for recursive function crawl
	 * 
	 * @param object 	$params
	 * @param array 	$params['domains']
	 * @param string	$params['entry']
	 * 
	 * @return boolean
	 *  
	 */
	public function crawl($params)
	{
		if (isset($params['domains']))
		{
			$this->domains = $params['domains'];	
		}
 
		if (isset($params['entry']))
		{
			$this->_crawl($params['entry']);	
 
			return true;
		}
 
		return false;		
	}
 
	/** 
	 * Recursive crawling function
	 * 
	 * @param string	$page
	 * @param int 		$parent [optional]
	 * @param int 		$depth of current page [optional]
	 * 
	 * @return void
	 * 
	 */
	protected function _crawl($page, $parent = 0, $depth = 0)
	{
		// vars
		$doc 	= '';
		$title 	= '';
		$desc  	= '';
		$out 	= NULL;
		$links 	= array();
		$link 	= '';
		//
 
		// correct url
		$page 	= $this->_buildUrl($page,$parent);
 
		// avoid reading the same url twice
		if (array_key_exists($page,$this->links))
		{
			return false;
		}
 
		// only read info for inner pages
		if (in_array($this->_getDomain($page),$this->domains))
		{			
			// read page content
			$doc 	= $this->_getHTTPRequest($page);
 
			// get page meta data
			list($title, $desc) = $this->_analyzePage($doc,$page);
		}
 
		// log into inner array
		$this->links[$page] = array(	'url' 		=> $page ,
						'title'		=> $title ,
						'description' 	=> $desc );
 
		// write into database		
		$this->ci->db->query('INSERT INTO web.map VALUES("",' .
					'"' . $page . '",' .
					'"' . $title . '",' .
					'"' . $desc . '",' .
					'"' . $parent . '") ');
 
		$parent = $this->ci->db->insert_id();	
 
		$this->_printLine($title,$desc,$page,$depth);	
 
 
		// avoid recursion for external domains
		if (!in_array($this->_getDomain($page),$this->domains))
		{
			return;
		}
 
		// now get links and launch recursively
		$links = $this->_getInnerLinks($doc);
 
		foreach($links as $link) 
		{ 
			if ($depth < $this->max_depth)
			{
				$this->_crawl($link,$parent,$depth+1);	
			}				
		}
 
		return;		
	}
 
	/**
	 * Prints the current links array
	 * 
	 *  
	 */
	public function printResults()
	{
		ksort($this->links);
 
		foreach ($this->links AS $link)
		{
			$this->_printLine($link['title'],$link['description'],$link['url'],0);
		}
	}
 
	/**
	 * Preloads the existent links in database 
	 * 
	 */
	private function _loadLinks()
	{
		$result = $this->ci->db->query('SELECT url,title,description FROM web.map ORDER BY url ASC');
 
		if ($result->num_rows())
		{
			$results = $result->result();
 
			foreach($results as $r)
			{
				$this->links[$r->url] = array(	'url' 		=> $r->url ,
								'title' 	=> $r->title ,
								'description' 	=> $r->description );		
			}
		}		 
	}
 
	/**
	 * Reads a document for html links
	 * 
	 * @param string $doc [optional]
	 * 
	 * @return array of links
	 *  
	 */
	private function _getInnerLinks($doc = '')
	{
		$return = array();
		$regexp = "<a\s[^>]*href=(\"??)([^\" >]*?)\\1[^>]*>(.*)<\/a>";
		$match  = null;
 
		if(preg_match_all("/$regexp/siU", $doc, $matches, PREG_SET_ORDER)) 
		{ 	
			foreach($matches as $match) 
			{ 
				$return[] = $match[2];
			} 
		}
 
		return $return;
	}
 
	/**
	 * Prints a line for the current url
	 * 
	 * @param int
	 * 
	 */
	private function _printLine($title = '', $desc = '', $page = '', $depth = 0)
	{
		// do not indent at this time
		$depth = 0;
		//
 
		echo '<p>' .
				'<h3 style="margin-left:' . ($depth*$this->indent) . 'px">' . 
					stripslashes($title) . 
				'</h3>' . 
				'<span style="margin-left:' . ($depth*$this->indent) . 'px">' .
					substr(stripslashes($desc),0,100) .
				'</span>' . 
				'<h4 style="margin-left:' . ($depth*$this->indent) . 'px">' . 
					$page . 
				'</h4>' .
			'</p>';	
	}	
 
	/**
	 * Analyzes a page to get meta information
	 * 
	 * @param string $doc(cument)
	 * @param string $page
	 * 
	 * @return list($title,$description);
	 *  
	 */
	private function _analyzePage($doc, $page)
	{
		//
		$title 	= '';
		$desc 	= '';
		//
 
		if (eregi ("<title>(.*)</title>", $doc, $out)) 
		{
			$title = addslashes($out[1]);	
 
			if (strlen($out[1]))
			{
				$title = substr($title,0,50);	
			}			
		}
 
		$out 	= @get_meta_tags($page);
 
		if (isset($out['description']))
		{
			$desc 	= addslashes($out['description']);
		}
 
		return array($title,$desc);
	}
 
	/**
	 * Completes a page link with the domain if needed
	 * 
	 * @param string $page
	 * @param int 	 $parent
	 * 
	 * @return string corrected url
	 *  
	 */
	private function _buildUrl($page,$parent)
	{
		// prepare url if relative
		if (!eregi('http://',$page))
		{
			// root relative path
			if (strpos($page,'/') == 0)
			{
				$page = 'http://' . $this->domains[0] . $page;	
			}
			// page relative path
			else
			{
				if (strpos($page,'mailto:')<0 && strpos($page,'javascript:')<0)
				{
					$page = $this->_getUrlByID($parent) . '/' . $page;	
				}				
			}			
		}
 
		// trim final slash
		$page = trim($page,'/');
 
		return $page;
	}
 
	/**
	 * Gets url for a given ID
	 * 
	 * @param int url id
	 * 
	 * @return string the url
	 *  
	 */
	private function _getUrlByID($id)
	{
		$results = $this->ci->db->query('SELECT url FROM web.map WHERE ID = "' . $id . '"');
 
		if ($results->num_rows())
		{
			$result = $results->result();
 
			return $result[0]->url;
		}
 
		return '';
	}
 
	/**
	 * Gets domain name from a URL
	 * 
	 * @param string url
	 * 
	 * @return string domain name
	 * 
	 */
	private function _getDomain($url = '')
	{
		// get host name from URL
		preg_match('@^(?:http://)?([^/]+)@i', $url, $matches);
 
		if (isset($matches[1]))
		{
			return $matches[1];	
		}
		else
		{
			return NULL;
		}
 
	}
 
 
	/**
	 * HTTP helper function.<br />
	 * Loads an http request and returns result.
	 *
	 * @param string $url to request
	 *
	 * @return string the result
	 *
	 */
	private function _getHTTPRequest($url = '')
	{
		// vars
		$html = '';
		$old  = '';
		$file = '';	
		//
 
		//configuration
		$timeout 			= 15;
		//
 
		// execution
		try
		{
 
  			$file = @fopen($url, 'rb');
 
			if ($file)
			{
				$html = stream_get_contents($file);
				fclose($file);
			}						
 
		}
		catch(Exception $e)
		{
			$html = 'error';
		}
		//
 
		return ($html);
	}
}