海量数据按行数进行切分

来源:互联网 发布:资金互助合作社软件 编辑:程序博客网 时间:2024/06/11 17:58
import osimport codecsdef split_file(read_file,write_file):    readfile = open(read_file,"r")    j = 0    for line in readfile:        line = line.strip()        line_str = line.split('\t')        if j%10000 == 0:  #10000条数据切分            num = int(j/10000) #设置文件的命名规则从0开始            _write_file = write_file+str(num)+".txt"            if os.path.exists(_write_file):                os.remove(_write_file)            writefile = open(_write_file,"w")        new_line = "'"+line_str[1]+"'"+","        writefile.write(new_line+ "\n")        j += 1    readfile.close()    writefile.close()if __name__=='__main__':    read_file = "C:\\Users\\ziyht\\Desktop\\mysql_data_mac.txt"    write_file = "C:\\Users\\ziyht\\Desktop\\test\\mac_"    split_file(read_file,write_file)
原创粉丝点击