pandas 读取大文件

来源:互联网 发布:win10系统图标网络关闭 编辑:程序博客网 时间:2024/06/05 04:00
from matplotlib import pyplot  import scipy as sp  import numpy as np  from matplotlib import pylab  from pandas import Series, DataFrameimport pandas as pdfilename = 'data.txt'data = pd.read_csv(filename, sep='::', names=list('uirt'), index_col=False, usecols=range(len('uirt')),engine = 'python', iterator=True)loop = TruechunkSize = 100000chunks = []while loop:    try:        chunk = data.get_chunk(chunkSize)        chunks.append(chunk)    except StopIteration:        loop = False        print "Iteration is stopped."data = pd.concat(chunks, ignore_index=True)
原创粉丝点击