python脚本之批量查询网站权重

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/6/2 13:50
# @Author  : 王先森
# @Blog    : www.boysec.cn
# @Software: PyCharm
# @Function :  批量查网站权重

import requests
import csv
from lxml import etree
import threading
from queue import Queue
import time
from urllib.parse import urlparse

header={'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0'}
class WebWeight(threading.Thread):
    def __init__(self,queue):
        threading.Thread.__init__(self)
        self.queue = queue
    def run(self):
        while not self.queue.empty():
            # 爱站的网站url
            aizhanurl = 'https://www.aizhan.com/cha/'
            # 获取查询的网站的url
            chaxunurl = self.queue.get()
            print("[+] 正在查询："+ chaxunurl)
            url = aizhanurl+chaxunurl
            time.sleep(4)  # 延迟4s
            re = requests.get(url, headers=header)
            print("[-] 请求url："+ url)
            html = re.text.encode(re.encoding).decode('utf-8')
            tree = etree.HTML(html)
            tags = tree.xpath('//a[@id="baidurank_br"]//@alt')

            # 写入数据
            # 如果使用python3就下面这个
            with open("webweight.csv", "a+", encoding='utf-8', newline='')as file:
            # 如果使用python2就下面这个
            # with open("webweight.csv", "a+")as file:
            # # 创建csv对象并传参
                csvwriter = csv.writer(file)
                for tag in tags:
                    # print(tag)
                    strtag = str(tag)  # 将tag属性转换为字符串
                    if strtag =='n' or strtag == '0':
                        csvwriter.writerow(['' + chaxunurl, '' + '0'])
                        print("[+] 查询结果: "+ chaxunurl+" 权重："+ str(0))
                    else:
                        csvwriter.writerow(['' + chaxunurl, '' + tag])
                        print("[+] 查询结果: " + chaxunurl + " 权重：" + tag)

def main():
    with open("webweight.csv", "a", encoding='utf-8', newline='')as file:
        csvwriter = csv.writer(file)
        csvwriter.writerow(['weburl','weight'])
    threads = []  # 线程集
    # 线程数
    threads_count = 5
    # 队列
    # 如果是使用python3就修改下面
   # queue = Queue()
    # queue = Queue.Queue()
    queue = Queue()
    with open("websites.txt", "r")as file:
        file_content = file.readlines()
        for i in file_content:
            # 文件读取中字符串结尾会有\r\n
            j = i.strip('\n').strip('\r')
            url = urlparse(j)
            if url.netloc:
                queue.put(url.netloc)
            else:
                queue.put(url.path)

    for i in range(threads_count):
        # 添加线程
        threads.append(WebWeight(queue))
    # 线程的开始与加入
    for i in threads:
        time.sleep(1)
        i.start()

    for i in threads:
        time.sleep(1)
        i.join()

    print("Results saved in webweight.csv")

if __name__ == '__main__':
    main()