爬虫学习--第六个爬虫程序

    #-*- coding:utf-8 -*-


from bs4 import BeautifulSoup

import requests

import os,sys

import time

class mzitu():

    def allUrl(self,url): #获取图片页面所有连接
        
        html = self.request(url)

        htmlSoup = BeautifulSoup(html.text,'lxml').find('div',class_='all').find_all('a')

        for x in htmlSoup:
            
            title = x.get_text()

            href = x['href'] #获取套图地址

            name = x['href'].split("/")[-1]

            
            print '##################################'
            print        u"开始保存..."+name+u"的套图"
            print '##################################'        
                   

            time.sleep(3)

            path = name

            chPath = "E:\py\%s" %name

            self.mkdir(path)

            os.chdir(chPath)

            self.html(href)

    def html(self,href): #处理套图的页面地址
        
        html = self.request(href)

        hrefSoup = BeautifulSoup(html.text,'lxml').find('div',class_='pagenavi').find_all('span')[-2].get_text()

        for page in range(1 , int(hrefSoup) + 1):

            pageUrl = href+'/'+str(page) #href是套图的地址 hrefSoup则是套图的页面地址

            self.img(pageUrl)

    def img(self,pageUrl): #处理获取图片实际地址
        
        imgUrl = self.request(pageUrl)

        imgSoup = BeautifulSoup(imgUrl.text,'lxml').find('div',class_='main-image').find('img')['src']

        self.save(imgSoup)

    def save(self,imgSoup): #下载保存图片

        print u"正在下载.."+imgSoup

        saveName = imgSoup.split("/")[-1]

        saveImg = self.request(imgSoup)

        f = open(saveName,'wb')

        f.write(saveImg.content)

        f.close()

    def mkdir(self,path): #创建文件夹
        
        path = path.strip()

        isExists = os.path.exists(os.path.join("E:\py", path))

        if not isExists:

            print(u"建了一个名字叫做" +path+ u"的文件夹！")

            os.makedirs(os.path.join("E:\py", path))

            return True
        else:

            print(u"名字叫做" +path+ u"的文件夹已经存在了！")
            
            return False       

    def request(self,url): #复用函数
    
        res = requests.get(url)

        return res


mzPic = mzitu()

mzPic.allUrl('http://www.mzitu.com/all')