caffe|Fine-tuning for driver

1 img–>lmdb

firstly we need the train.txt,and test.txt

import ospath='/path/path/'#we can also use os.walk(dir)def imgname_to_txt(dir,output_txt):    output=open(output_txt,mode='w')#mode:write read add    files=os.listdir(dir)    for name in files:        fullname=os.path.join(dir,name)        if (os.path.isdir(fullname)):            subfiles=os.listdir(fullname)            for subname in subfiles:                output.write(name+'/'+subname+' '+fullname[-1]+'\n')        else:            output.write(name+' '+'0'+'\n')    output.close()imgname_to_txt(path+'/img/train',path+'/data/train.txt')imgname_to_txt(path+'/img/test',path+'/data/test.txt')

and maybe we want a val data set:

import numpy as nptrain=open(path+'/data/train.txt','r')train_lines=train.readlines()l=len(train_lines)print ltrain.close()train_sub=open(path+'/data/train_sub.txt','w')val=open(path+'/data/val.txt','w')np.random.seed(0)randomval=np.random.choice(range(l),int(l*0.3),replace=False)#replace=False !print len(randomval)print randomval[:5]for i in randomval:    val.write(train_lines[i])val.close()train_index=[i for i in range(l) if i not in randomval]print train_index[:5]print len(train_index)for j in train_index:    train_sub.write(train_lines[j])train_sub.close()

let convert img to lmdb with $TOOLS/convert_imageset, with the sh file in imagenet :

echo "Creating train lmdb..."GLOG_logtostderr=1 $TOOLS/convert_imageset \    --resize_height=$RESIZE_HEIGHT \    --resize_width=$RESIZE_WIDTH \    --shuffle \    $TRAIN_DATA_ROOT \    $DATA/train_sub.txt \    $EXAMPLE/driver_train_sub_lmdb

we need to compute image mean:

$TOOLS/compute_image_mean $EXAMPLE/**_train_lmdb \  $DATA/imagenet_mean.binaryproto

draw_net

./python/draw_net.py --rankdir TB /media/beatree/file/note_for_project/work/conpetitions/kaggle/driver/dirver_predict_img.prototxt  /media/beatree/file/note_for_project/work/conpetitions/kaggle/driver/driver_caffnet2.png

2 train

./build/tools/caffe train -solver /media/beatree/file/note_for_project/work/conpetitions/kaggle/driver/solverall.prototxt -weights /home/beatree/caffe-rc3/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel -gpu 0

3 predict

3.1 binaryproto->npy

http://blog.csdn.net/hyman_yx/article/details/51732656

import caffeimport numpy as npMEAN_PROTO_PATH = 'mean.binaryproto'               # 待转换的pb格式图像均值文件路径MEAN_NPY_PATH = 'mean.npy'                         # 转换后的numpy格式图像均值文件路径blob = caffe.proto.caffe_pb2.BlobProto()           # 创建protobuf blobdata = open(MEAN_PROTO_PATH, 'rb' ).read()         # 读入mean.binaryproto文件内容blob.ParseFromString(data)                         # 解析文件内容到blobarray = np.array(caffe.io.blobproto_to_array(blob))# 将blob中的均值转换成numpy格式，array的shape （mean_number，channel, hight, width）mean_npy = array.mean(0)                                np.save(MEAN_NPY_PATH ,mean_npy)# print mean_npy

3.2 classification

mport numpy as npimport matplotlib.pyplot as plt# display plots in this notebook%matplotlib inline# set display defaultsplt.rcParams['figure.figsize'] = (10, 10)        # large imagesplt.rcParams['image.interpolation'] = 'nearest'  # don't interpolate: show square pixelsplt.rcParams['image.cmap'] = 'gray'  # use grayscale output rather than a (potentially misleading) color heatmappath='/pathpath/driver/'import caffecaffe.set_device(0)caffe.set_mode_gpu()model_def = path + 'dirver_predict_img.prototxt'model_weights = path+'model/finetune_iter_20000.caffemodel'net = caffe.Net(model_def,      # defines the structure of the model                model_weights,  # contains the trained weights                caffe.TEST)     # use test mode (e.g., don't perform dropout)# load the mean ImageNet image (as distributed with Caffe) for subtractionmu = np.load(path + 'data/mean.npy')mu = mu.mean(1).mean(1)  # average over pixels to obtain the mean (BGR) pixel valuesprint 'mean-subtracted values:', zip('BGR', mu)# create transformer for the input called 'data'transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})transformer.set_transpose('data', (2,0,1))  # move image channels to outermost dimensiontransformer.set_mean('data', mu)            # subtract the dataset-mean value in each channeltransformer.set_raw_scale('data', 255)      # rescale from [0, 1] to [0, 255]transformer.set_channel_swap('data', (2,1,0))  # swap channels from RGB to BGR# set the size of the input (we can skip this if we're happy#  with the default; we can also change it later, e.g., for different batch sizes)net.blobs['data'].reshape(32,        # batch size                          3,         # 3-channel (BGR) images                          227, 227)  # image size is 227import pandas as pdimport numpy as npsample_df=pd.read_csv(path+'sample_submission.csv')submission_df=pd.DataFrame(columns=sample_df.columns,index=sample_df.index)submission_df.img=sample_df.imgbatch_size=32start_index=0sample_len=len(sample_df)while start_index<sample_len:    end_index=min(start_index+batch_size,sample_len)    print 'predicting',start_index,':',end_index    img=[caffe.io.load_image(path+'img/test/{}'.format(sample_df.img[i]))for i in range(start_index,end_index)]    transformed_image =[ transformer.preprocess('data', img[i])for i in range (len(img)) ]    if end_index<sample_len:        net.blobs['data'].data[0:32]=transformed_image    else:        net.blobs['data'].data[0:(end_index%32)] = transformed_image    output = net.forward()    if end_index<sample_len:        output_prob = output['probs'] [0:32]    else:        output_prob = output['probs'] [0:(end_index%32)]    print output_prob.argmax(1)    submission_df.iloc[range(start_index,end_index),1:]=output_prob    start_index=end_indexsubmission_df.to_csv(path+'submission.csv',index=False)print 'done'

result