def open_spider(self, spider):
hostname = '192.168.12.130'
username = 'postgres'
password = 'postgres'
database = 'weibo'
# 创建连接
self.connection = psycopg2.connect(host=hostname, user=username, password=password, dbname=database)
self.cur = self.connection.cursor()
def process_item(self, item, spider):
try:
self.cur.execute("INSERT INTO weibospider(mid,numberpage,isspider,creattime) VALUES (%s,%s,%s,now());",
(item['mid'], item['number_page'], item['is_spider']))
except Exception as e:
log.msg(e, level=log.ERROR)
return item
def close_spider(self, spider):
self.cur.close()
self.connection.close()
pipelinesITEM_PIPELINES = {
'WeiBoSpider.pipelines.WeibospiderPipeline': 300,
}
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。