Laravel Symfony_Crawler GuzzleHttp 爬虫抓取行政区域

2018/05/29 修改抓取编码gb2312改gb18030

项目需要行政区域三级联动，刚好写个爬虫练练手。

Laravel 框架，安装的两个库

 composer require guzzlehttp/guzzle
 composer require symfony/dom-crawler

创建表

DROP TABLE IF EXISTS `area`;
CREATE TABLE `area` (
  `id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
  `name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `parent_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

DROP TABLE IF EXISTS `crawler`;
CREATE TABLE `crawler` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `status` int(11) DEFAULT '0',
  `data` text COLLATE utf8mb4_unicode_ci,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

创建对应Model

App\Model\Area.php

<?php

namespace App\Model;

use Illuminate\Database\Eloquent\Model;

class Area extends Model
{
    public $timestamps = false;

    protected $table = 'area';

    protected $keyType = 'string';

    protected $fillable = [
        'id', 'name', 'parent_id',
    ];
}

App\Model\Crawler.php

<?php

namespace App\Model;

use Illuminate\Database\Eloquent\Model;

class Crawler extends Model
{
    public $timestamps = false;

    protected $table = 'crawler';

    protected $fillable = [
        'id', 'status', 'data'
    ];
}

app/Console/Kernel.php 添加

    protected $commands = [
        'App\Console\Commands\CityCrawler',  
    ];

新建 App\Console\Commands\CityCrawler.php

<?php
namespace App\Console\Commands;

use Illuminate\Console\Command;
use Symfony\Component\DomCrawler\Crawler;

use App\Model\Area;
use App\Model\Crawler as CrawlerTask;

use GuzzleHttp\Psr7;
use GuzzleHttp\Exception\RequestException;

// 流程：
//     1. func top 抓取行政区域省级, 每个省链接生成一次抓取任务，保存到任务表crawler。
//     2. 循环抓取 
//             1). 读取一条任务 
                    select * from crawler where status = 0 limit 1; 
                    update crawler set status = 1 where id = 本次任务id;
//             2). 根据任务类型调用不同抓取方法 如：
                     镇级：crawler_towntr 
                     区级：crawler_districts 
                     城市：crawler_citys。
//                 保存抓取到数据到area表，并生成子级区域抓取任务。
class CityCrawler extends Command
{

    protected $signature = 'city:crawler';
    protected $description = 'City Crawler';
    protected $start_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/';
    protected $special_city = ['东莞市','中山市','嘉峪关市','三沙市','儋州市']; // 中国5个不设市辖区的地级市 

    public function handle()
    {
        //抓取省级行政区域
        $this->top(); 

        while (true) {
            $task_model = $this->task();

            if(empty($task_model)){
                return $this->info("End");
            }
            $task= json_decode($task_model->data, true);

            // 打印日志
            $this->info(implode(',', array_map(function($item){
                return $item['id'] . ' ' . $item['name'];
            }, $task['data'])));

            $status = call_user_func(array($this, 'crawler_' . $task['crawler']), $task);
            if($status){
                $this->finish($task_model);
            }else{
                var_dump($task, 'error');
                return false;
            }
            $this->info("sleep 1");
            sleep(1);
        }
    }


    public function finish($task){
        $task->status = 2;
        $task->save();
    }

    public function task($status = 0)
    {
        $task = CrawlerTask::where("status", $status)->first();
        $task->status = 1; // 进行中
        $task->save();
        return $task;
    }

    public function push($data)
    {
        $task = new CrawlerTask;
        $task->data = json_encode($data);
        $task->save();
    }

    // 第一个页面
    public function top()
    {
        $url = $this->start_url;

        $html = $this->send_http($url);

        $crawler = new Crawler();
        $crawler->addHtmlContent($html, 'gb18030');
        $crawler->filter('.provincetr')->filter('td > a')->each(function(Crawler $node, $i) use($url) {
            $text = $node->text();
            $href = $node->attr('href');
            $id = str_replace('.html', '', $href);
            
            $task = [
                'crawler' => 'citys',
                'remark' => '省',
                'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
                'data' => [ [ 'name' => $text, 'id' => $id] ],
                'parent_id' => $id,
            ];
            $this->push($task);

            Area::create(
                [
                    'id' => $id,
                    'name' => $text,
                    'parent_id' => 0
                ]
            );

            $this->info($node->attr('href'));
            $this->info($text);
        });
    }


    public function crawler_towntr($task)
    {

        $url = $task['url'];

        if(!strpos($url, '.html')){
            $this->info('为空的直辖市');
            return true;
        }

        $html = $this->send_http($url);

        $crawler = new Crawler();
        $crawler->addHtmlContent($html, 'gb18030');
        $crawler->filter('.towntr')->each(function(Crawler $node, $i) use ($task, $url) {

            $code_node = $node->filter('td')->eq(0)->filter('a');
            $name_node = $node->filter('td')->eq(1)->filter('a');

            Area::create(
                [
                    'id' => $code_node->text(),
                    'name' => $name_node->text(),
                    'parent_id' => $task['parent_id']
                ]
            );

            $this->info($code_node->text() . '  ' . $name_node->text());
        });

        return true;
    }

    public function crawler_districts($task)
    {
        $url = $task['url'];
        $html = $this->send_http($url);

        $crawler = new Crawler();
        $crawler->addHtmlContent($html, 'gb18030');
        $crawler->filter('.countytr')->each(function(Crawler $node, $i) use ($task, $url) {


            $code_node = $node->filter('td')->eq(0)->filter('a');
            $name_node = $node->filter('td')->eq(1)->filter('a');
            
            //没有子节点
            if($code_node->count() == 0){   
                $code_node = $node->filter('td')->eq(0);
                $name_node = $node->filter('td')->eq(1);
            }else{

                $href = $code_node->attr("href");
                $data = $task['data'];
                $data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ;

                $new_task = [
                    'crawler' => 'towntr',
                    'remark' => '县 区',
                    'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
                    'data' => $data,
                    'parent_id' => $code_node->text(),
                ];

                $this->push($new_task);                
            }

            $this->info($code_node->text() . '  ' . $name_node->text());

            Area::create(
                [
                    'id' => $code_node->text(),
                    'name' => $name_node->text(),
                    'parent_id' => $task['parent_id']
                ]
            );

        });
        return true;
    }

    public function crawler_citys($task)
    {
        $url = $task['url'];
        $html = $this->send_http($url);

        $crawler = new Crawler();
        $crawler->addHtmlContent($html, 'gb18030');
        $crawler->filter('.citytr')->each(function(Crawler $node, $i) use ($task, $url) {

            $code_node = $node->filter('td')->eq(0)->filter('a');
            $name_node = $node->filter('td')->eq(1)->filter('a');
            $href = $code_node->attr("href");

            $this->info($code_node->text() . '  ' . $name_node->text());

            Area::create(
                [
                    'id' => $code_node->text(),
                    'name' => $name_node->text(),
                    'parent_id' => $task['parent_id']
                ]
            );
            

            $data = $task['data'];
            $data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ;

            if(in_array($name_node->text(), $this->special_city)){
                $new_task = [
                    'crawler' => 'towntr',
                    'remark' => '特别的5个省地级市',
                    'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
                    'data' => $data,
                    'parent_id' => $code_node->text(),
                ];                
            }else{

                $new_task = [
                    'crawler' => 'districts',
                    'remark' => '城市',
                    'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
                    'data' => $data,
                    'parent_id' => $code_node->text(),
                ];

            }


            $this->push($new_task);

        });

        return true;
    }

    public function info($string, $verbosity = null)
    {
        $string = iconv( 'UTF-8', 'GB18030', $string); // cmd 中文gbk编码
        parent::line($string, 'info', $verbosity);
    }

    private function send_http($url)
    {   
        $user_agent_list = [
            'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04',
            'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.9 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52',
            'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0',
            'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
            'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
        ];

        $user_agent = $user_agent_list[(time() % 6)];
        $timeout = 5; // 秒

        $client = new \GuzzleHttp\Client(['headers' => ['User-Agent' => $user_agent], 'timeout' => $timeout]);

        try {
            $res  = $client->request('GET', $url);
            $html =  (string)$res->getBody();

        } catch (RequestException $e) {
            // 抓取中会有404状态返回，再重新请求一次。
            $this->info(Psr7\str($e->getRequest()));
            if ($e->hasResponse()) {
                $this->info(Psr7\str($e->getResponse()));
            }

            $this->info("send_http timeout retry");
            $this->info("sleep 2s");
            sleep(2);
            $res  = $client->request('GET', $url);
            $html =  (string)$res->getBody();              
        }
        
        return $html;
    }

}

进目录运行

php artisan city:crawler

最后

------------------------------------------------------

数据有了全部写到一个json文件里，太大了1M多 :(

还是写成ajax从服务端读取三级联动数据。

area.js

// 1. 省加载 其他请选择
// 2. 省 change 触发加载市
// 3. 市触发加载区
// 4. 区加载触发街道
function area_init(param) {
    var area = this;
    area.area_not_filter = false;;
    area.prompt_html = '<option value="">－请选择－</option>';
    if (param.area_not_filter) {
        area.area_not_filter = param.area_not_filter;
    }
    if (param.province) {
        area.province_el = $(param.province);
    }
    if (param.city) {
        area.city_el = $(param.city);
    }
    if (param.district) {
        area.district_el = $(param.district);
    }
    if (param.street) {
        area.street_el = $(param.street);
    }
    area.load = function() {
        area.province_el.html(area.prompt_html);
        area.city_el.html(area.prompt_html);
        area.district_el.html(area.prompt_html);
        if (area.street_el) {
            area.street_el.html(area.prompt_html);
        }
        province_id = area.province_el.attr("data-value");
        city_id = area.city_el.attr("data-value");
        district_id = area.district_el.attr("data-value");
        if (area.street_el) {
            street_id = area.street_el.attr("data-value");
        }
        area.area_fill(0, 'province', area.province_el, province_id);
        province_id && area.area_fill(province_id, 'city', area.city_el, city_id);
        city_id && area.area_fill(city_id, 'district', area.district_el, district_id);
        if (area.street_el) {
            district_id && area.area_fill(district_id, 'street', area.street_el, street_id);
        }
    }
    area.bind = function() {
        area.province_el.change(function() {
            area.area_fill($(this).val(), 'city', area.city_el);
            area.city_el.html(area.prompt_html);
            area.district_el.html(area.prompt_html);
            if (area.street_el) {
                area.street_el.html(area.prompt_html);
            }
        });
        area.city_el.change(function() {
            area.area_fill($(this).val(), 'district', area.district_el);
            area.district_el.html(area.prompt_html);
            if (area.street_el) {
                area.street_el.html(area.prompt_html);
            }
        });
        if (area.street_el) {
            area.district_el.change(function() {
                area.area_fill($(this).val(), 'street', area.street_el);
                area.street_el.html(area.prompt_html);
            });
        }
    };
    area.area_fill = function(parent_id, level, el, active_id) {
        // value='' 不请求  ajax
        if (parent_id === '') {
            return false;
        }
        area.get_area(parent_id, level, function(list) {
            var province = area.prompt_html;
            $.each(list, function(i, n) {
                province += '<option ' + (active_id == n.id ? ' selected ' : ' ') + ' value="' + n.id + '">' + n.name + '</option>';
            });
            $(el).html(province);
        });
    }
    area.get_area = function(parent_id, level, callback) {
        var $url = '/ajax_area/' + level + '/' + parent_id;
        $.ajax({
            url: $url,
            type: 'get',
            success: function(res) {
                callback(res);
            }
        });
    }
    area.load();
    area.bind();
}

// 使用方法
// data-value 默认值
// 32 江苏省
// 320100000000 南京市
// 320102000000 玄武区
// <select name="province" data-value="32"></select>
// <select name="city" data-value="320100000000"></select>
// <select name="district" data-value="320102000000"></select>
// <select name="street" data-value=""></select>

// var area = new area_init(
//         {
//             province: "select[name='province']",
//             city: "select[name='city']",
//             district: "select[name='district']",
//             street: "select[name='street']"
//         }  
//     );

// 或者

// <select name="province" data-value="32"></select>
// <select name="city" data-value="320100000000"></select>

// var area = new area_init(
//         {
//             province: "select[name='province']",
//             city: "select[name='city']"
//         }  
//     );

// 如果需要js动态修改
// $('select[name="province"]').attr('data-value', data.province);
// $('select[name="city"]').attr('data-value', data.city);
// $('select[name="district"]').attr('data-value', data.district);
// area.load();

成品长这样哈

Laravel Symfony_Crawler GuzzleHttp 爬虫抓取行政区域