重难点 | Python基础知识点9 Python的封装继承多态，推荐收藏

class Spider(object):

class Spider(object):
# 定义一个抽象类
    __metaclass__ = abc.ABCMeta

def __init__(self):
        self.rows_title = [u'招聘标题', u'公司名称', u'公司地址', u'待遇', u'发布日期', u'招聘链接', u'招聘要求描述']
        sheet_name = u'51job_Python招聘'
        return_execl = ExeclUtils.create_execl(sheet_name, self.rows_title)
        self.execl_f = return_execl[0]
        self.sheet_table = return_execl[1]
        self.job_info = []  # 存放每一条数据中的各元素，
        self.count = 0  # 数据插入从1开始的

def crawler_data(self):
'''
        开始爬取数据
        :return:
        '''
for i in range(1, 5):
            url = 'http://search.51job.com/list/000000,000000,0000,00,9,99,python,2,{}.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='.format(
                i)
            self.request_job_list(url)
# 采集不要太快了，否则容易造成ip被封或者网络请求失败
            time.sleep(2)

def request_job_list(self, page_url):
'''
        获取工作列表
        :param page_url:
        :return:
        '''
try:
            headers = {
'Referer': 'http://www.51job.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4549.400 QQBrowser/9.7.12900.400'
            }
            response = requests.get(page_url, headers=headers)
            response.encoding = 'gbk'
# 如果请求失败，则不能继续进行
if response.status_code != 200:
return
            self.parse_job_list(response.text)
except Exception as e:
print '\n\n出现错误,错误信息是:{}\n\n'.format(e.message)

    @abc.abstractmethod
def parse_job_list(self, text):
'''
        解析工作列表的抽象类，具体实现在子类中
        :param text:
        :return:
        '''
pass

def request_job_detail(self, job_href):
'''
        获取工作详情
        :param job_href: 招聘工作的链接
        :return:
        '''
try:
            headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4549.400 QQBrowser/9.7.12900.400'
            }
            response = requests.get(job_href, headers=headers)
            response.encoding = 'gbk'
# 如果请求失败，则不能继续进行
if response.status_code != 200:
return ''

            self.parse_job_detail(response.text)

except Exception as e:
print '\n\n出现错误,错误信息是:{}\n\n'.format(e.message)

    @abc.abstractmethod
def parse_job_detail(self, text):
'''
        定义工作详情的抽象类
        :param text:
        :return:
        '''
pass

  
# coding:utf-8
from bs4 import BeautifulSoup
from ExeclUtils import ExeclUtils
from Spider import Spider
import time


class SpiderBs4(Spider):

def __init__(self):
        super(SpiderBs4, self).__init__()

def parse_job_list(self, text):
try:
            soup = BeautifulSoup(text, 'html.parser')
            results = soup.select('div.dw_table > div.el')[1:]
for result in results:
                job_title = result.select('p.t1 span a')
                job_href = result.select('p.t1 span a')
                job_company = result.select('span.t2  a')
                job_address = result.select('span.t3')
                job_salary = result.select('span.t4')
                job_date = result.select('span.t5')

                job_title = job_title[0].attrs['title'] if len(job_title) > 0 else ''
                job_href = job_href[0].attrs['href'] if len(job_href) > 0 else ''
                job_company = job_company[0].attrs['title'] if len(job_company) > 0 else ''
                job_address = job_address[0].text if len(job_address) > 0 else ''
                job_salary = job_salary[0].text if len(job_salary) > 0 else ''
                job_date = job_date[0].text if len(job_date) > 0 else ''

                self.job_info.append(job_title)
                self.job_info.append(job_company)
                self.job_info.append(job_address)
                self.job_info.append(job_salary)
                self.job_info.append(job_date)
                self.job_info.append(job_href)

                self.request_job_detail(job_href)
                time.sleep(1)
except Exception as e:
print '\n\n出现错误,错误信息是:{}\n\n'.format(e.message)

def parse_job_detail(self, text):
try:
            soup = BeautifulSoup(text, 'html.parser')
try:
# 工作描述
                job_statements = soup.select('div.job_msg')
                job_statement = job_statements[0].text.strip(' ').replace(' ', '').replace('\n', '')
except Exception as e:
print e.message
                job_statement = '职位无明确描述'

            self.job_info.append(job_statement)
            self.count = self.count + 1
            ExeclUtils.write_execl(self.execl_f, self.sheet_table, self.count, self.job_info, u'bs4_51job招聘.xlsx')
print '采集了{}条数据'.format(self.count)
# 清空集合,为再次存放数据做准备
            self.job_info = []
except Exception as e:
print '\n\n出现错误,错误信息是:{}\n\n'.format(e.message)

#
#
# if __name__ == '__main__':
#     x = SpiderBs42()
#     x.crawler_data()

  @abc.abstractmethod
def parse_job_list(self, text):
'''
        解析工作列表的抽象类，具体实现在子类中
        :param text:
        :return:
        '''
pass
        
      @abc.abstractmethod
def parse_job_detail(self, text):
'''
        定义工作详情的抽象类
        :param text:
        :return:
        '''
pass