Scrapy爬虫，华为商城商品数据爬虫demo

python -m venv ven

cd huawei
env\Scripts\activate.bat

scrapy startproject vmall_spider

cd vmall_spider

scrapy genspider -t crawl vmall "vmall.com"

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from vmall_spider.items import VmallSpiderItem

class VamllSpider(CrawlSpider):    
 name = 'vmall'    
 allowed_domains = ['vmall.com']    
 start_urls = ['https://www.vmall.com/']    
 rules = (        
  Rule(LinkExtractor(allow=r'.*/product/.*'), callback='parse_item', follow=True),    
  )   

 def parse_item(self, response):
  title=response.xpath("//div[@class='product-meta']/h1/text()").get()     
  image=response.xpath("//a[@id='product-img']/img/@src").get()        
  item=VmallSpiderItem(
   title=title,
   image=image,
  )
  print("="*30)        
  print(title)
  print(image)        
  print("="*30)        
  yield item
    

import scrapy


class VmallSpiderItem(scrapy.Item):
    title=scrapy.Field()
    image=scrapy.Field()
    

import pymysql
import os
from urllib import request


class VmallSpiderPipeline:    
 def __init__(self):        
  dbparams={
      'host':'127.0.0.1', #云数据库弹性公网IP
      'port':3306, #云数据库端口            
      'user':'vmall', #云数据库用户
      'password':'123456', #云数据库RDS密码
      'database':'vmall', #数据库名称            
      'charset':'utf8'        
  }        
  self.conn=pymysql.connect(**dbparams)        
  self.cursor=self.conn.cursor()        
  self._sql=None

  self.path=os.path.join(os.path.dirname(os.path.dirname(__file__)),'images')        
  if not os.path.exists(self.path):            
   os.mkdir(self.path)    


 def process_item(self,item,spider):        
  url=item['image']        
  image_name=url.split('_')[-1]        
  print("--------------------------image_name-----------------------------")        
  print(image_name)        
  print(url)        
  request.urlretrieve(url,os.path.join(self.path,image_name))        
  self.cursor.execute(self.sql,(item['title'], item['image']))        
  self.conn.commit()        
  return item  
    

BOT_NAME = 'vmall_spider'

SPIDER_MODULES = ['vmall_spider.spiders']
NEWSPIDER_MODULE = 'vmall_spider.spiders'

ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 3

DEFAULT_REQUEST_HEADERS = {
 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',    
 'Accept-Language': 'en',    
 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',    
 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',    
 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16'
}

ITEM_PIPELINES = {
 'vmall_spider.pipelines.VmallSpiderPipeline': 300,
}

from scrapy import cmdline
cmdline.execute("scrapy crawl vmall".split())