携程酒店真实房价抓取

Python编程与实战

发布于 2019-12-04 15:47:10

1.8K0

发布于 2019-12-04 15:47:10

文章被收录于专栏：Python编程与实战

携程的反爬机制有点让人害怕，当某些参数不对的时候，直接返回的房价价格比实际价格要高，这也可以当作“千人千面”吧。

阅读步骤

效果展示
项目地址
携程反爬机制图示
携程反爬三点
总结

效果展示

项目地址

https://github.com/justcodedroid/spider_js/tree/master/xiecheng

携程反爬机制图示

携程反爬三点

我们抓取的地址是这类的。https://hotels.ctrip.com/hotel/7067729.html

抓取数据是乱码（温馨提示不要使用json中html字段），我们可以通过debugjs函数发现，字体之间有一个映射函数。用node重写。

function parser(e) {
    e = Buffer.from(e, 'base64').toString()
    function t(e, t, o) {
        var i = "";
        try {
            i = ((n, t) => {

                var r, o, e = "1", i = void 0 == e[0], c = i ? [] : "";
                for (r = 0; r < n.length; r++)
                    o = t.charAt(n.charAt(r).charCodeAt(0) - 21760).charAt(0),
                        i ? c.push(o) : c += o;
                return c = i ? c.join("") : c
            })(e, t)
        } catch (n) {
            i = ""
        }
        return i
    }
    var i = ["T", "F", "e", "s", "M", "a", "g", "r", "o", "B", "u", "l", "y", "m", "f", "t"];
    e = JSON.parse(e)
    e.html = t(e.ComplexHtml, e.ASYS,)
    return Buffer.from(e.html).toString('base64')
}

获取eleven参数，这个参数可以理解成浏览器的指纹。他会采集很多细节，然后拼成一串字符串。这段代码是用nodejs写了一个服务。

function get_eleven(text, url) {
    text = new Buffer(text, 'base64').toString()
    url = new Buffer(url, 'base64').toString()
    let process = require('process')
    process.exit = function () {
        throw  'fuck'
    }
    let JSDOM = require('jsdom').JSDOM
    dom = new JSDOM('<html><head><body>hh</body></head></html>')
    location = {'href': url}
    document = dom.window.document
    window = {location: location, document: document}
    navigator = {
        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
        geolocation: {}
    }
    function Image() {
    }
    eleven = null
    function CASrYVcraFQgjGciKk(f) {
        eleven = f()
    }
    eval(text)
    return Buffer.from(eleven).toString('base64')
}
module.exports = get_eleven

cookie的加密。ferror,_zQdjfing这两个值的生成xmind中已经介绍了，ferror依赖hotalid+UA,_zQdjfing依赖ferror。最坑的是hashCode方法，依赖首页的4个变量。

大家可以先用正则提取4个变量，然后调用下面方法。

function get_hash_code(c,a,s,m){
tc = []
md = function (n, m, k) {
    var p;
    if (t[m])
        return t[m];
    if (m == 1) {
        t[m] = n % k;
        return t[m]
    }
    if (m & 1) {
        p = (m - 1) >> 1;
        t[1] = n % k;
        t[m] = (md(n, p, k) + md(n, p, k) + n % k) % k;
        return t[m]
    }
    p = m >> 1;
    t[m] = (md(n, p, k) + md(n, p, k)) % k;
    return t[m]
}
rd = function () {
    t = [0];
    s = (md(a, s, m) + c % m) % m;
    return s
}
for (var i = 0; i < 256; i++) {
    var r = '0x';
    for (var j = 0; j < 8; j++) {
        r += (rd() % 16).toString(16)
    }
    tc.push(parseInt(r, 16))
}

function hashCode() {
    var _0x24e50e = {
        'pmqAv': function _0x3bc67a(_0x21f6a6, _0xeed6f) {
            return _0x21f6a6 < _0xeed6f;
        },
        'tUguD': function _0x529e47(_0x6476ea, _0x5ee855) {
            return _0x6476ea ^ _0x5ee855;
        },
        'bTlKh': function _0x4d3092(_0x4010ff, _0xde8cef) {
            return _0x4010ff ^ _0xde8cef;
        },
        'PFtAy': function _0x59e07b(_0x3240f1, _0x482071) {
            return _0x3240f1 & _0x482071;
        },
        'DppDN': function _0x580566(_0x3bb80b, _0x142af4) {
            return _0x3bb80b >> _0x142af4;
        },
        'WavRi': function _0x1cb726(_0x320a0b, _0x1acb93) {
            return _0x320a0b & _0x1acb93;
        },
        'EcTAI': function _0x22ee49(_0x3277f3, _0x196702) {
            return _0x3277f3 >> _0x196702;
        },
        'ZjAoT': function _0x1c129a(_0x52c557, _0x5722fa) {
            return _0x52c557 & _0x5722fa;
        }
    };
    var _0x328279 = _0x3eceb6 = this.length
        , _0x5da81b = 0x0;
    for (; _0x24e50e.pmqAv(_0x5da81b, _0x3eceb6); _0x5da81b++) {
        var _0x238d5f = this.charCodeAt(_0x5da81b);
        if (_0x238d5f >> 0x8) {
            _0x328279 = _0x24e50e[_0x4f05('0x7', '\x6f\x79\x59\x26')](_0x328279 >> 0x8, tc[_0x24e50e['\x62\x54\x6c\x4b\x68'](_0x24e50e[_0x4f05('0x8', '\x6a\x66\x4d\x25')](_0x328279, 0xff), _0x24e50e[_0x4f05('0x9', '\x42\x73\x36\x74')](_0x238d5f, 0x8))]);
            _0x238d5f = _0x24e50e[_0x4f05('0xa', '\x21\x21\x23\x33')](_0x238d5f, 0xff);
        }
        _0x328279 = _0x24e50e.EcTAI(_0x328279, 0x8) ^ tc[_0x24e50e.bTlKh(_0x24e50e.ZjAoT(_0x328279, 0xff), _0x238d5f)];
    }
    return _0x328279;
}
return hashCode
}

调用示例，给String添加个hashCode方法
c = 0x269ec3,
a = 0x343fd,
s = 1552287018864,// 这几个值是前端网页动态生成的，最好定时更新。
m = 0x7FFFFFFF
String.prototype.hashCode = get_hash_code(c,a,s,m)