怎么用mysql保存数据
如题,求详细代码
如题,求详细代码
2019-05-23
import MySQLdb from douban.settings import mysql_host, mysql_port, mysql_user, mysql_passwd, mysql_db_name # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class DoubanPipeline(object): def __init__(self): host = mysql_host port = mysql_port user = mysql_user passwd = mysql_passwd dbname = mysql_db_name self.mydb = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=dbname, charset='utf8') self.cursor = self.mydb.cursor() def process_item(self, item, spider): data = dict(item) sql = "insert into top250 (serial_number, movie_name, introduce, star, evaluate, description) values (%s,%s,%s,%s,%s,%s)" self.cursor.execute(sql, (data['serial_number'], data['movie_name'], data['introduce'], data['star'], data['evaluate'], data['description'])) self.mydb.commit() return item
seetings中
#mysql的基本参数 sql_host = 'localhost' sql_db_name = 'spider' sql_sheetname = 'douban' sql_user='root' sql_password=''
pipelines中
#构造函数用于初始化mysql的各项参数 def __init__(self): host = sql_host dbname =sql_db_name ruser = sql_user rpassword = sql_password self.sheetname = sql_sheetname #连接数据库 self.conn = pymysql.connect(host=host, user=ruser, password=rpassword, db=dbname, charset='utf8') # 创建一个游标 self.cursor = self.conn.cursor() #防止连接出现错误 def open_spider(self,spider): try: host = sql_host dbname = sql_db_name ruser = sql_user rpassword = sql_password self.sheetname = sql_sheetname # 连接数据库 self.conn = pymysql.connect(host=host, user=ruser, password=rpassword, db=dbname, charset='utf8') # 创建一个游标 self.cursor = self.conn.cursor() except: self.open_spider() else: spider.logger.info('MySQL: connected') self.cursor = self.conn.cursor(pymysql.cursors.DictCursor) spider.cursor = self.cursor def process_item(self, item, spider): #item是从douban_spider里传出的数据 #先将数据转换为字典形式 data = dict(item) #mysql插入数据 keys = ','.join(data.keys()) values = ','.join(['%s'] * len(data)) sql = 'INSERT INTO {table}({keys}) VALUES({values})'.format(table=self.sheetname, keys=keys, values=values) try: self.cursor.execute(sql,tuple(data.values())) self.conn.commit() except: self.conn.rollback() #self.cursor.close() #self.conn.close() #mongodb插入数据 #self.post.insert(data) return item def close_spider(self,spider): self.cursor.close() self.conn.close()
注意 存储中字段不能叫describe,好像是python中的关键字,改为describes或其他
举报