python小程序:把名称列表从csv文件读入mongo,再从mongo导入redis(anaconda3.5.2)
来源:互联网 发布:驾照模拟考试c1软件 编辑:程序博客网 时间:2024/06/04 17:50
#/usr/bin/env python
import redis
import csv
import pymongo
def readDictCSV(fileName="E:\\comp.csv", keyIndex=2):
with open(fileName, 'U') as csvfile:
reader = csv.reader(csvfile)
# print(reader,type(reader))
column = [row[10] for row in reader]
return column[1:]
def list2mongo(mongo_conn, db_name='com_db', collection_name='com_col', field_name='company_name', comlist=None):
collection = mongo_conn[db_name][collection_name]
collection.ensure_index(field_name, unique=True)
for a_com in comlist:
try:
collection.insert({field_name:a_com})
except pymongo.errors.DuplicateKeyError:
continue
def mongo_chk(mongo_conn, db_name='com_db', collection_name='com_col', field_name='company_name'):
collection = mongo_conn[db_name][collection_name]
for a_com in collection.find():
print(a_com)
print("total: ", collection.count())
def mongo2redis(mongo_conn, rds, key='com_rlist', db_name='com_db', collection_name='com_col', field_name='company_name'):
collection = mongo_conn[db_name][collection_name]
for a_com in collection.find():
rds.lpush(key, a_com[field_name])
def redis_chk(rds, key='com_rlist'):
cnt = rds.llen(key)
coms = rds.lrange(key,0,-1)
print(coms)
print('total: ',cnt)
def redis_trim_all(rds, key='com_rlist'):
rds.ltrim(key,0,0)
rds.flushall()
cnt = rds.llen(key)
def pop_comlist(rds, key='com_rlist'):
len = rds.llen(key)
while 1:
lst, name = rds.brpop(key)
if name==None:
break
else:
# print(name,type(name))
a = name.decode('utf-8')
print(a)
return len
def main():
m_conn=pymongo.MongoClient('10.0.0.9', 27018)
pool=redis.ConnectionPool(host='10.0.0.225', port=6379)
r = redis.Redis(connection_pool=pool)
coms = readDictCSV()
list2mongo(mongo_conn=m_conn,db_name='com_db', collection_name='com_col', field_name='company_name',comlist=coms)
mongo_chk(mongo_conn=m_conn, db_name='com_db', collection_name='com_col', field_name='company_name')
mongo2redis(mongo_conn=m_conn, rds=r, key='com_rlist', db_name='com_db', collection_name='com_col', field_name='company_name')
redis_chk(rds=r, key='com_rlist')
pop_comlist(rds=r, key='com_rlist')
redis_trim_all(r, key='com_rlist')
redis_chk(rds=r, key='com_rlist')
if __name__ == '__main__':
main()
import redis
import csv
import pymongo
def readDictCSV(fileName="E:\\comp.csv", keyIndex=2):
with open(fileName, 'U') as csvfile:
reader = csv.reader(csvfile)
# print(reader,type(reader))
column = [row[10] for row in reader]
return column[1:]
def list2mongo(mongo_conn, db_name='com_db', collection_name='com_col', field_name='company_name', comlist=None):
collection = mongo_conn[db_name][collection_name]
collection.ensure_index(field_name, unique=True)
for a_com in comlist:
try:
collection.insert({field_name:a_com})
except pymongo.errors.DuplicateKeyError:
continue
def mongo_chk(mongo_conn, db_name='com_db', collection_name='com_col', field_name='company_name'):
collection = mongo_conn[db_name][collection_name]
for a_com in collection.find():
print(a_com)
print("total: ", collection.count())
def mongo2redis(mongo_conn, rds, key='com_rlist', db_name='com_db', collection_name='com_col', field_name='company_name'):
collection = mongo_conn[db_name][collection_name]
for a_com in collection.find():
rds.lpush(key, a_com[field_name])
def redis_chk(rds, key='com_rlist'):
cnt = rds.llen(key)
coms = rds.lrange(key,0,-1)
print(coms)
print('total: ',cnt)
def redis_trim_all(rds, key='com_rlist'):
rds.ltrim(key,0,0)
rds.flushall()
cnt = rds.llen(key)
def pop_comlist(rds, key='com_rlist'):
len = rds.llen(key)
while 1:
lst, name = rds.brpop(key)
if name==None:
break
else:
# print(name,type(name))
a = name.decode('utf-8')
print(a)
return len
def main():
m_conn=pymongo.MongoClient('10.0.0.9', 27018)
pool=redis.ConnectionPool(host='10.0.0.225', port=6379)
r = redis.Redis(connection_pool=pool)
coms = readDictCSV()
list2mongo(mongo_conn=m_conn,db_name='com_db', collection_name='com_col', field_name='company_name',comlist=coms)
mongo_chk(mongo_conn=m_conn, db_name='com_db', collection_name='com_col', field_name='company_name')
mongo2redis(mongo_conn=m_conn, rds=r, key='com_rlist', db_name='com_db', collection_name='com_col', field_name='company_name')
redis_chk(rds=r, key='com_rlist')
pop_comlist(rds=r, key='com_rlist')
redis_trim_all(r, key='com_rlist')
redis_chk(rds=r, key='com_rlist')
if __name__ == '__main__':
main()
阅读全文
0 0
- python小程序:把名称列表从csv文件读入mongo,再从mongo导入redis(anaconda3.5.2)
- [bigdata-039] pysql+pymongo+py3把数据从sql 导入 到mongo的通用代码
- 【python/热力图】从csv文件读入(x,y,z)画简单热力图
- mongo 导入json文件
- python小程序:以后台服务方式 定时 监视redis队列 (anaconda3.5.2)
- 把数据库文件写入csv文件和从csv文件中导入数据到数据库中
- [bigdata-112] mongo3.x数据从一个公网导出然后再导入阿里mongo
- Python之csv文件从MySQL数据库导入导出
- 从mongo得到数据分页
- mongo导入
- 从键盘读入一行字符,把其中的字母字符一次存放在磁盘文件f2.dat中。再把它从磁盘文件读入程序,将其中的小写字母改为大写字母,再存入磁盘文件f3.dat
- 从csv文件导入数据到postgres
- Python Mongo
- python mongo
- python从文件中读入中文字符
- C++ 读入csv 文件程序
- php redis mongo python 数据类型小结
- Java从入门到精通——数据库篇Mongo DB 导出,导入,备份
- python小程序:以后台服务方式 定时 监视redis队列 (anaconda3.5.2)
- 用消息系统实现分布式事务
- 【Shell脚本(二)】运算符,命令和文件
- C#关于数组排序
- 1036. 跟奥巴马一起编程(15)
- python小程序:把名称列表从csv文件读入mongo,再从mongo导入redis(anaconda3.5.2)
- Go基础编程:自定义函数
- 时间换算
- Python yield 斐波那契数列
- RHEL各个版本的技术容量与限制
- 关于REST API的为什么
- mac-Linux一般操作
- 操作系统总结一
- 衷心感谢你的光临