PHP实现网页URL爬虫

<?php 

class CrawlUrl{

  /**
   * 从给定的url获取html内容
   * @param string $url 
   * @return array 
   */
  public static function crawler($url) {
    $content = self::_getUrlContent($url);
    if ($content) {
      $url_list = self::_reviseUrl($url, self::_filterUrl($content));
      if ($url_list) {
        return $url_list;
      } else {
        return ;
      } 
    } else {
      return ;
    } 
  } 


  /**
   * 获取HTML内容
   * @param string $url 
   * @return string 
   */
  private static function _getUrlContent($url) {
    $handle = @fopen($url, "r");
    if ($handle) {
      // 读取已打开的资源流到一个字符串
      $content = stream_get_contents($handle, 1024 * 1024);
      return $content;
    } else {
      return false;
    } 
  } 


  /**
   * 从html内容中筛选链接
   * @param string $web_content 
   * @return array 
   */
  private static function _filterUrl($web_content) {
    $reg_tag_a = '/<[a|A].*?href=[\'\"]{0,1}([^>\'\"\ ]*).*?>/';
    // 在html内容中筛选出链接
    $result = preg_match_all($reg_tag_a, $web_content, $match_result);
    if ($result) {
      return $match_result[1];
    } 
  } 


  /**
   * 修正相对路径
   * @param string $base_url 
   * @param array $url_list 
   * @return array 
   */
  private static function _reviseUrl($base_url, $url_list) {
    /*
     * 解析 URL，返回其组成部分
     * http://username:password@hostname/path?arg=value#anchor
     * [scheme] => http
     * [host] => hostname
     * [user] => username
     * [pass] => password
     * [path] => /path
     * [query] => arg=value
     * [fragment] => anchor
    */
    $url_info = parse_url($base_url);
    $base_url = $url_info["scheme"] . '://';
    if ($url_info["user"] && $url_info["pass"]) {
      $base_url .= $url_info["user"] . ":" . $url_info["pass"] . "@";
    } 
    $base_url .= $url_info["host"];
    if ($url_info["port"]) {
      $base_url .= ":" . $url_info["port"];
    } 
    $base_url .= $url_info["path"];
    if (is_array($url_list)) {
      foreach ($url_list as $url_item) {
        if (preg_match('/^http/', $url_item)) {
          // 已经是完整的url
          $result[] = $url_item;
        } else {
          // 不完整的url
          $real_url = $base_url . '/' . $url_item;
          $result[] = $real_url;
        } 
      } 
      return $result;
    } else {
      return;
    } 
  } 

}

echo "<pre>";
$current_url = "https://www.cnblogs.com/shamohai/p/11158206.html"; //初始url
print_r($result_url_arr = CrawlUrl::crawler($current_url));

// 记录url列表 只写方式打开 将文件指针指向文件末尾
$fp_puts = fopen("url.txt", "ab"); 
// 保存url列表 只读方式打开 将文件指针指向文件头
$fp_gets = fopen("url.txt", "r"); 
if ($result_url_arr) {
  foreach ($result_url_arr as $url) {
     // fwrite() 的别名
    fputs($fp_puts, $url . "\r\n");
  } 
}