selenium实战-抓取百度网盘分享链接

def createDriver():
    """
    每次关闭Chromedriver后，重新创建一个
    :return:
    """
    # 创建driver，并指定Chromedriver的路径
    current_dir = os.path.abspath('.')
    chromedriverPath = os.path.join(current_dir, 'chromedriver')
    browser = webdriver.Chrome(executable_path=chromedriverPath)

    return browser

def getInfo(file_name, keyword):
    key_wd = {'wd': keyword}
    # URL encode
    target_url = Format_str + urllib.parse.urlencode(key_wd)

    # 启动浏览器
    browser = createDriver()
    browser.get(target_url)
    sleep(2)

    #切换到百度知道tab
    #//*[@id="s_tab"]/div/a[4]
    browser.find_element_by_xpath('//*[@id="s_tab"]/div/a[4]').click()
    sleep(2)

    # 搜索结果链接,第一个基本上就是百度知道的结果，所有只取第一页10条，如果搜不到直接放弃空字符
    for index in range(1, 11):
        # 获取详情页xpath
        prefix_xpath = '//*[@id="wgt-list"]/dl[' + str(index) + ']'

        # 判断该条搜索结果是否为百度知道
        isBaiduKown(file_name=file_name, browser=browser, prefix=prefix_xpath)

    # 关闭浏览器
    browser.quit()

def getResultDetail(file_name, browser):
    """
    :param browser: 浏览器
    :return: 文件百度网盘地址详情
    """
    # 回答总个数统计
    answer_number_text = browser.find_element_by_xpath('//*[@id="qb-content"]/div[2]/span[2]').get_attribute(
        'innerHTML')
    answer_number = answer_number_text.split('个')[0]

    # 当答案大于2个时，答案列表会折叠
    if int(answer_number) > 2:
        # 更多按钮
        more_button_xpath = '//*[@id="show-answer-hide"]'
        if isElementExist(by=By.XPATH,value=more_button_xpath,driver=browser) == True:
            # 若存在更多按钮，先按展开所有答案
            browser.find_element_by_xpath(more_button_xpath).click()

        # 更多折叠按钮
        more_hide_xpath = '//*[@id="wgt-answers"]/div/div[6]/div'
        if isElementExist(by=By.CLASS_NAME,value='pager-next',driver=browser) == True and int(answer_number) > 5:
            if browser.find_element_by_class_name('pager-next').is_displayed() == False:
                browser.find_element_by_xpath(more_hide_xpath).click()
        else:
            browser.find_element_by_xpath('//*[@id="wgt-answers"]/div/div[5]/div').click()

        # 每页5个答案
        page_num = int(answer_number) // 5 + 1
        for page_index in range(1, page_num):
            # 获取答案详情，并判断是否有网盘下载链接
            # 最佳答案
            if page_index == 1:
                best_answer = browser.find_element_by_class_name('best-text.mb-10').text
                if downloadLinkObserve(file_name=file_name, detail_text=best_answer, browser=browser) == True:
                    browser.quit()
                    break

            # 普通答案
            nomal_answers_elements = browser.find_elements_by_class_name('answer-text.mb-10.line')
            for nomal_element in nomal_answers_elements:
                answer_text = nomal_element.text
                if downloadLinkObserve(file_name=file_name, detail_text=answer_text, browser=browser) == True:
                    browser.quit()
                    break

            # 如果有下一页，在未获取到下载链接的情况下，跳转到下一页继续
            if page_index < page_num and int(answer_number) > 5:
                browser.find_element_by_class_name('pager-next').click()

    # 切换到上一页的列表
    switchWindow(browser=browser, windows_signal='current')

def findUrl(string):
    """
    提取符串中URL链接
    :param string:
    :return:
    """
    # findall() 查找匹配正则表达式的字符串
    import re

    url = "https:" + re.match(r"[^/]+(/[^ ]*)", string).group(1)
    return url

def downloadLinkObserve(file_name, detail_text, browser):
    """
    检查回答中是否有百度网盘下载链接，如果有，再检查该链接分享是否已经失效
    :param detail_text:
    :param browser:
    :return:
    """
    if 'pan.baidu.com/s' in detail_text:
        download_url = findUrl(detail_text)
        download_code = ''
        if ("提取码" in detail_text):
            download_code = detail_text.split("提取码")[1]
        if (requestIsValid(url=download_url, browser=browser)) == True:
            file_link = {'download': download_url, 'code': download_code}

            # 更新数据库数据
            data.dbDeal.update_result(fileLink=str(file_link), fileName=file_name)

            return True
    else:
        return False