python读取csv大数据文件到mysql数据库中(ubunu14.04下)

来源:互联网 发布:红包源码 编辑:程序博客网 时间:2024/05/20 23:33
#- * - coding: UTF-8 -*-
import mysql.connector
import csv
#数据库连接
config={
        'host':'127.0.0.1',#默认127.0.0.1
        'user':'root',
        'password':'ict',
        'port':3306 ,#默认即为3306
        'database':'tianyi',
        'charset':'utf8'#默认即为utf8
        }
try:
  cnn=mysql.connector.connect(**config)
  if cnn:
      print 'ok'
except mysql.connector.Error as e:
  print('connect fails!{}'.format(e))
#处理
cursor=cnn.cursor()
#插入数据
#读取用户csv的文件
read = csv.reader(open('tianchi_mobile_recommend_train_user.csv'))
count = 0
datas = []
for useid,idtem,behavior,ugeohash,cate,time in read:
    sql_insert="insert into tian_yi_user(user_id,item_id,behavior_type,user_geohash,item_category,time) values (%(user_id)s,%(item_id)s,%(behavior_type)s,%(user_geohash)s,%(item_category)s,%(time)s)"
    data = {'user_id':useid,'item_id':idtem,'behavior_type':behavior,'user_geohash':ugeohash,'item_category':cate,'time':time}
    datas.append(data)
    count+=1
    if count/200000==1:
        cursor.executemany(sql_insert,datas)
        cnn.commit()
        count=0
        datas=[]
cursor.executemany(sql_insert,datas)
cnn.commit()
cursor.close()

cnn.close() 

#读取商品csv的文件

read = csv.reader(open('tianchi_mobile_recommend_train_item.csv'))
for idtem,geohash,category in read:
   sql_insert="insert into tian_yi_item(item_id,item_geohash,item_category) values (%(item_id)s,%(item_geohash)s,%(item_category)s)"
    data = {'item_id':idtem,'item_geohash':geohash,'item_category':category}
    cursor.execute(sql_insert,data)
cnn.commit()
cursor.close()

cnn.close()

读取文件数据时注释掉其中一个。

0 0