文本预处理小结

#coding=gbk'''Created on 2016-01-14 08:56:27@author: heng'''import reimport datetimestart = datetime.datetime.now()file1 = open('I:\\test1\\8. Tweet and Retweets with indexed content\\weibocontents\\Retweet_content.txt','r')file2 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test_final1.txt','w')rule3 = '^@.*(\n)$'rule2 = '^(retweet).*\n'rule1 = '^(link).*(\n)$'compile_rule1 = re.compile(rule1)compile_rule2 = re.compile(rule2)compile_rule3 = re.compile(rule3)for i in file1:    i1 = ''.join(i)    i2 = compile_rule1.sub('',i1)    i3 = compile_rule2.sub('',i2)    i4 = compile_rule3.sub('',i3)    file2.write(str(i4))file1.close()file2.close()end = datetime.datetime.now()print end-startprint 'program is ended!!!!!!!!'

#coding=gbk'''Created on 2016-01-14 15:08:49抽取转发微博相关数据@author: heng'''import datetimedef retweet_content(file1,file2):    j = 0;k=0    for i in file1:        j = j+1        i1  = ''.join(i)        i2 = i1.strip('\n').split('\t')        if len(i2)==4:            str_0 = ''.join(i2[0])         if len(i2)==3:            file2.write('\t'.join(i2)+'\t'+str_0)            k = j        if  j==k+1:            file2.write('\t'+str(i1))    file1.close()    file2.close()def time_processing(file3,file4):    for i in file3:        i1=''.join(i)        i2 = i1.split('\t')        i3 = ''.join(i2[1])        i3 = i3[0:10]+' '+i3[11:19]        i2[1] = i3        i4 = '\t'.join(i2)                file4.write(str(i4))    file3.close()    file4.close()start = datetime.datetime.now()file1  = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test_final1.txt','r')file2 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test3.txt','w')retweet_content(file1, file2)file3 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test3.txt','r')file4 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test4.txt','w')time_processing(file3,file4)end = datetime.datetime.now()print end-startprint 'program is ended!!!!!!!!'