专栏：015：重构“你要的实战篇"

- 文章标题：`Title` 
- 文章发布时间：`Time` 
- 文章全文：`Content` 
- 文章的链接：`Url` 思路：

- ~~分析首页和翻页的组成~~
- ~~抓取全部的文章链接~~
- ~~在获取的全部链接的基础上解析需要的标题，发布时间，全文和链接~~

start_urls = ["http://mindhacks.cn/"]
rules = (Rule(SgmlLinkExtractor(allow=(r'http://mindhacks.cn/page/\d+/',))),
         Rule(SgmlLinkExtractor(allow=(r'http://mindhacks.cn/\d{4,}/\d{2,}/\d{2,}/.*?-.*?-.*?-.*?/')), callback='parse_detail', follow = True)
         )
# 前一个Rule获取的是1,2,3,4页的网页组成： 如：http://mindhacks.cn/page/2/
# 后一个Rule获取的1,2,3,4网页下符合要求的文章的链接, 再在获取的文章链接的基础上进行解析 如：http://mindhacks.cn/2009/07/06/why-you-should-do-it-yourself/

def parse_detail(self, response):
   Item = LiuweipengItem()
   selector = Selector(response)
   title = selector.xpath('//div[@id="content"]/div/h1[@class="entry-title"]/a/text()').extract()
   time = selector.xpath('//div[@id="content"]/div/div[@class="entry-info"]/abbr/text()').extract()
   content = selector.xpath('//div[@id="content"]/div/div[@class="entry-content clearfix"]/p/text()').extract()
   url = selector.xpath('//div[@id="content"]/div/h1[@class="entry-title"]/a/@href').extract()
   for title, time, content, url in zip(title, time, content, url):
       Item["Title"] = title
       Item["Time"] = time
       Item["Content"] = content
       Item["Url"] = url
   yield Item
# 返回的Item 是需要抓取字段

from sqlalchemy import Column, String, Integer
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Article(Base):
    __tablename__ = "article"
    id = Column(Integer, primary_key=True)
    Title = Column(String)
    Time = Column(String)
    Content = Column(String)
    Url = Column(String)

CREATE TABLE `article` (
  `id` INT(11) NOT NULL AUTO_INCREMENT,
  `Title` VARCHAR(255) COLLATE utf8_bin NOT NULL,
  `Content` VARCHAR(255) COLLATE utf8_bin NOT NULL,
  `Time` VARCHAR(255) COLLATE utf8_bin NOT NULL,
  `Url` VARCHAR(255) COLLATE utf8_bin NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=INNODB AUTO_INCREMENT=39 DEFAULT CHARSET=utf8 COLLATE=utf8_bin

def open_spider(self, spider):
    engine = create_engine("mysql://root:123456@localhost:3306/test?charset=utf8", echo = True)
    DBSession = sessionmaker(bind=engine)
    self.session = DBSession()
    pass

def process_item(self, item, spider):
    one = Article(Title=item["Title"],
                Time=item["Time"],
                Content=item["Content"],
                Url=item["Url"])
    self.session.add(one)

    pass
def close_spider(self, spider):
    self.session.commit()
    self.session.close()
    pass

# 文件中添加如下代码
from scrapy.cmdline import execute
execute("scrapy crawl name".split())