Dogs vs. Cats Redux: Kernels Edition
来源:互联网 发布:淘宝驳回售后服务 编辑:程序博客网 时间:2024/06/15 19:45
训练样本数量超过内存,采用python生成器
基于keras + tensorflow+Resnet-34
直接上代码
#coding=utf-8import numpy as npimport osimport refrom keras.preprocessing.image import load_img,img_to_arrayfrom keras.models import Modelfrom keras.layers import Conv2D,AveragePooling2D,Dense,BatchNormalization,Flatten,add,Input,MaxPooling2D,ZeroPadding2Dfrom keras.optimizers import SGDfrom keras.callbacks import ModelCheckpointfrom sklearn.preprocessing import LabelEncoderseed = 7np.random.seed(seed)def getFileList(dir,fileList): if os.path.isfile(dir): fileList.append(dir.decode('gbk')) elif os.path.isdir(dir): for s in os.listdir(dir): newDir = os.path.join(dir, s) getFileList(newDir, fileList) return fileListdef Conv2d_BN(x, nb_filter,kernel_size, strides=(1,1), padding='same'): x = Conv2D(nb_filter,kernel_size,padding=padding,strides=strides,activation='relu')(x) x = BatchNormalization(axis=3)(x) return xdef Conv_Block(inpt,nb_filter,kernel_size,strides=(1,1), with_conv_shortcut=False): x = Conv2d_BN(inpt, nb_filter=nb_filter, kernel_size=kernel_size, strides=strides, padding='same') x = Conv2d_BN(x, nb_filter=nb_filter, kernel_size=kernel_size, padding='same') if with_conv_shortcut: shortcut = Conv2d_BN(inpt, nb_filter=nb_filter, strides=strides, kernel_size=kernel_size) x = add([x, shortcut]) return x else: x = add([x, inpt]) return xdef Resnet(): inpt = Input(shape=(224,224,3)) x = ZeroPadding2D((3, 3))(inpt) x = Conv2d_BN(x, nb_filter=64, kernel_size=(7, 7), strides=(2, 2), padding='valid') x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) # (56,56,64) x = Conv_Block(x, nb_filter=64, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=64, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=64, kernel_size=(3, 3)) # (28,28,128) x = Conv_Block(x, nb_filter=128, kernel_size=(3, 3), strides=(2, 2), with_conv_shortcut=True) x = Conv_Block(x, nb_filter=128, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=128, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=128, kernel_size=(3, 3)) # (14,14,256) x = Conv_Block(x, nb_filter=256, kernel_size=(3, 3), strides=(2, 2), with_conv_shortcut=True) x = Conv_Block(x, nb_filter=256, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=256, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=256, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=256, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=256, kernel_size=(3, 3)) # (7,7,512) x = Conv_Block(x, nb_filter=512, kernel_size=(3, 3), strides=(2, 2), with_conv_shortcut=True) x = Conv_Block(x, nb_filter=512, kernel_size=(3, 3)) x = Conv_Block(x, nb_filter=512, kernel_size=(3, 3)) x = AveragePooling2D(pool_size=(7, 7))(x) x = Flatten()(x) x = Dense(1, activation='sigmoid')(x) model = Model(inpt,x) sgd = SGD(lr=0.001, momentum=0.9) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) model.summary() return modeldef generateData(batch_size): label = ['cat','dog'] encoder = LabelEncoder() encoder.fit(label) train_img = getFileList(r'/media/wmy/document/BigData/kaggle/DogsvsCats/train', []) while True: np.random.shuffle(train_img) batch = 0 X = [] y = [] for path in train_img: img = load_img(path, target_size=(224, 224)) img = img_to_array(img) label = re.findall(r'.+?/train/([a-z]+)', path)[0] X.append(img) y.append(label) batch+=1 if batch % batch_size==0: batch = 0 yield (np.array(X),np.array(encoder.transform(y))) X = [] y = []def generateTestData(batch_size): #读取顺序不是按存储顺序 test_img = getFileList(r'/media/wmy/document/BigData/kaggle/DogsvsCats/test', []) while True: batch = 0 X = [] for path in test_img: img = load_img(path, target_size=(224, 224)) img = img_to_array(img) X.append(img) batch += 1 if batch % batch_size == 0: batch = 0 yield (np.array(X)) X = []def train_model(): callback = [] model_check = ModelCheckpoint(filepath='best_params.h5',monitor='acc',save_best_only=True,mode='max') callback.append(model_check) model = Resnet() model.fit_generator(generator=generateData(20),steps_per_epoch=25000,epochs=50,verbose=1,callbacks=callback,max_q_size=4)def pred(): test_img = getFileList(r'/media/wmy/document/BigData/kaggle/DogsvsCats/test', []) numbers = [] for path in test_img: numb = re.findall(r'.+?/test/([0-9]+)', path)[0] numbers.append(int(numb)) model = Resnet() model.load_weights('best_params.h5') ypred = model.predict_generator(generator=generateTestData(20),steps=12500,max_q_size=10,verbose=2) ypred_index_data = np.c_[numbers,ypred] ypred_index_data = list(ypred_index_data) ypred_index_data.sort(key=lambda x:x[0]) np.savetxt("my_pred.csv", ypred_index_data, delimiter=',', header='id,label', comments='',fmt=['%d','%f'])if __name__=='__main__': train_model() pred()
0 0
- Dogs vs. Cats Redux: Kernels Edition
- Kaggle Dogs vs. Cats Redux: Kernels Edition (猫狗大战)by Pytorch
- kaggle之Dogs vs. Cats(Keras)
- rain cats and dogs
- Keras+Theano后端,CPU、GPU、cuDNN加速对比(Dogs vs. Cats和mnist)
- 基于TensorFlow的Cats vs. Dogs(猫狗大战)实现和详解(1)
- 基于TensorFlow的Cats vs. Dogs(猫狗大战)实现和详解(2)
- Kaggle : Using a Convolutional Neural Network for classifying Cats vs Dogs
- Pytorch实战指南---使用Pytorch完成Kaggle上的经典比赛:Dogs vs Cats---updating
- kernels
- Q3.7 Create a data structures to hold dogs and cats.(待续)
- Redux
- Redux
- Redux
- redux
- redux
- Redux
- Redux
- 3Sum
- C#项目5.4
- vbs脚本demo
- Hibernate分页实现
- java.lang.NoClassDefFoundError: org/apache/log4j/Level的解决方案
- Dogs vs. Cats Redux: Kernels Edition
- 对称矩阵的基本操作及其压缩存储
- Form表单传递List数组属性到后台对象中
- Python实现互联网笔试题-今日头条-3个一组的试题
- *** glibc detected *** malloc(): memory corruption(一)
- mysql索引
- Android动画之View Animation(视图动画)
- Java API 小结
- 值得深思的面试题