记录快速傅里叶变换计算相关面并附tensorflow相应层代码

Fast Template Matching,J. P. Lewis

#-*- coding:utf-8 -*-import numpy as npimport matplotlib.pyplot as pltimport cv2from Integral_image import *## read img #####tm_path = '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/1_AB_real_B.png'ref_path = '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/1_AB_real_B.png'img_ref_bgr = cv2.imread(ref_path)img_tm_bgr = cv2.imread(tm_path)img_ref_gray = (cv2.cvtColor(img_ref_bgr,cv2.COLOR_BGR2GRAY))/np.float64(255.0)img_src_gray = (cv2.cvtColor(img_tm_bgr,cv2.COLOR_BGR2GRAY))/np.float64(255.0)[h_ref,w_ref] = img_ref_gray.shape[h_tm,w_tm] = img_src_gray.shapeh_tm = int(h_tm/3)w_tm = int(w_tm/3)ex_ratio = 0img_tm_gray_ex = np.zeros([h_ref+h_tm*ex_ratio,w_ref+w_tm*ex_ratio])img_tm_gray = np.copy(img_src_gray[h_tm:h_tm+h_tm,w_tm:w_tm+w_tm])img_tm_gray_ex[0:h_tm,0:w_tm] = np.copy(img_src_gray[h_tm:h_tm+h_tm,w_tm:w_tm+w_tm])tm_sum = np.sum(img_tm_gray_ex[0:h_tm,0:w_tm])tm_sum_sq = np.sum(img_tm_gray_ex[0:h_tm,0:w_tm]*img_tm_gray_ex[0:h_tm,0:w_tm])img_tm_gray_ex[0:h_tm,0:w_tm] = img_tm_gray_ex[0:h_tm,0:w_tm] - tm_sum/(h_tm*w_tm)  #去均值in_ref = calcIntag(img_ref_gray)in_ref_sq = calcIntag(img_ref_gray*img_ref_gray)img_ref_gray_ex = np.zeros([h_ref+h_tm*ex_ratio,w_ref+w_tm*ex_ratio])img_ref_gray_ex[0:h_ref,0:w_ref] = img_ref_gray[:,:]# img_ref_gray_ex[0:h_ref-h_tm+1,0:w_ref-h_tm+1] = img_ref_gray_ex[0:h_ref-h_tm+1,0:w_ref-h_tm+1] #- ref_avg_mat  #去均值#傅里叶变换F_tm = np.fft.fft2(-img_tm_gray_ex)F_tm.real = -F_tm.realF_ref = np.fft.fft2(img_ref_gray_ex)F_w = F_tm * F_refres = np.fft.ifft2(F_w).real#/((h_tm)*(w_tm))res_i = np.fft.ifft2(F_w).imagprint(res_i)bg_h = 0bg_w = 0ref_div_sq = calcDivmat2(in_ref_sq,in_ref, h_tm, w_tm)tm_div = tm_sum_sq - (1.0/(h_tm*w_tm))*tm_sum*tm_sumres = res[bg_h:bg_h+h_ref-h_tm+1,bg_w:bg_w+w_ref-w_tm+1]/(np.sqrt(ref_div_sq*tm_div))print(np.max(res),np.min(res))print(img_ref_gray.dtype,img_tm_gray.dtype)res2 = cv2.matchTemplate(np.float32(img_ref_gray),np.float32(img_tm_gray),method=5)print(np.max(res2),np.min(res2))plt.figure()plt.subplot(221),plt.imshow(res,cmap='gray')plt.subplot(222),plt.imshow(res2,cmap='gray')plt.subplot(223),plt.imshow(in_ref,cmap='gray')plt.subplot(224),plt.imshow(in_ref_sq,cmap='gray')plt.show()

tensorflow 层

#-*- coding:utf-8 -*-import tensorflow as tfimport cv2import numpy as npimport matplotlib.pyplot as pltfrom Integral_image import *## read img #####tm_path = '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/1_AB_real_B.png'ref_path = '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/1_AB_real_B.png'img_ref_bgr = cv2.imread(ref_path)img_tm_bgr = cv2.imread(tm_path)img_ref_gray = (cv2.cvtColor(img_ref_bgr,cv2.COLOR_BGR2GRAY))/255.0img_src_gray = (cv2.cvtColor(img_tm_bgr,cv2.COLOR_BGR2GRAY))/255.0[h_ref,w_ref] = img_ref_gray.shape[h_tm,w_tm] = img_src_gray.shapeh_tm = int(h_tm/3)w_tm = int(w_tm/3)sz_tm = h_tm*w_tmsz_ratio = 1.0/(sz_tm)#img_tm_gray = np.zeros([h_ref,w_ref])img_tm_gray = np.copy(img_src_gray[h_tm:h_tm+h_tm,w_tm:w_tm+w_tm])###################################build compute graph#################tensor_tm_src = tf.convert_to_tensor(img_tm_gray,dtype=tf.float32)tensor_ref_src = tf.convert_to_tensor(img_ref_gray,dtype=tf.float32)Weight_tm = tf.Variable(tf.ones(img_tm_gray.shape),dtype=tf.float32)Weight_ref = tf.Variable(tf.ones(img_ref_gray.shape),dtype=tf.float32)tensor_tm = Weight_tm*tensor_tm_srctensor_ref = Weight_ref*tensor_ref_src#计算积分图，均值等##tm的均值和平方均值tm_sum = tf.reduce_sum(tensor_tm)tm_sum_sq = tf.reduce_sum(tensor_tm*tensor_tm)##ref的积分图和平方积分图tensor_ref_sq = tensor_ref*tensor_ref#站位，在外面计算ref_integral_img = tf.placeholder(dtype=tf.float32,shape=(h_ref,w_ref))ref_integral_img_sq = tf.placeholder(dtype=tf.float32,shape=(h_ref,w_ref))ref_div_sq_mat = tf.placeholder(dtype=tf.float32,shape=(h_ref-h_tm+1,w_ref-w_tm+1))##模板去均值tensor_tm = tensor_tm - tm_sum/sz_tm##拼成一个大的tensortensor_left = tf.zeros((h_tm,w_ref-w_tm),dtype=tf.float32)tensor_bottom = tf.zeros((h_ref-h_tm,w_ref),dtype=tf.float32)tensor_tm_ex = tf.concat([tf.concat([tensor_tm,tensor_left],axis=1),tensor_bottom],axis=0)###fft计算卷积F_tm = tf.fft2d(tf.complex(-tensor_tm_ex,tf.zeros(tensor_tm_ex.shape)))F_tm = -tf.conj(F_tm)F_ref = tf.fft2d(tf.complex(tensor_ref,tf.zeros(tensor_ref.shape)))#F_m = tf.multiply(F_ref,F_tm)F_m = F_ref * F_tmres = tf.real(tf.ifft2d(F_m))[0:h_ref-h_tm+1,0:w_ref-w_tm+1]#计算分母tm_div = tm_sum_sq - sz_ratio*tm_sum*tm_sumres = res/tf.sqrt(ref_div_sq_mat*tm_div)#训练设置optmizer = tf.train.AdamOptimizer(0.05)loss = -res[h_tm,w_tm]train = optmizer.minimize(loss)init = tf.global_variables_initializer()###################################################run gragh ##############with tf.Session() as sess:    sess.run(init)    for i in range(20):        tm_img_show = sess.run(tensor_tm)        ref_img_show = sess.run(tensor_ref)        ref_arr_sq= sess.run(tensor_ref_sq)        ref_arr = sess.run(tensor_ref)        in_ref_sq = calcIntag(ref_arr_sq)        in_ref = calcIntag(ref_arr)        ref_div_sq = calcDivmat2(in_ref_sq,in_ref,h_tm,w_tm)        sess.run(train, feed_dict={ref_integral_img: in_ref, ref_integral_img_sq: in_ref_sq, ref_div_sq_mat: ref_div_sq})    loss_out,res_out= sess.run([loss,res],feed_dict={ref_integral_img:in_ref,ref_integral_img_sq:in_ref_sq,ref_div_sq_mat:ref_div_sq})    print(loss_out,np.max(res_out),np.min(res_out))    #验证    res2 = cv2.matchTemplate(np.float32(img_ref_gray),np.float32(img_tm_gray),5)    print(np.max(res2),np.min(res2))    plt.figure()    plt.subplot(221),plt.title('tm'),plt.imshow(tm_img_show,cmap='gray')    plt.subplot(222),plt.title('ref'), plt.imshow(ref_img_show, cmap='gray')    plt.subplot(223), plt.imshow(res2, cmap='gray')    plt.subplot(224), plt.imshow(res_out,cmap='gray')    plt.figure()    plt.imshow(res_out-res2,cmap='gray',vmax=1.0,vmin=-1.0)    plt.show()

支持batch的层，fft2d不知道能不能直接计算batch，用for太慢了

#-*- coding:utf-8 -*-import tensorflow as tffrom Integral_image import *class NCC_Layer(object):    def __init__(self,tensor_ref,tensor_tm,h_ref,w_ref,h_tm, w_tm,batch_size,channal):        self.tensor_ref = tensor_ref   #[batch,h_ref,w_ref,channal]        self.tensor_tm = tensor_tm     #[batch,h_tm,w_tm,channal]        self.h_ref = h_ref        self.w_ref = w_ref        self.h_tm = h_tm        self.w_tm = w_tm        self.sz_tm = h_tm*w_tm        self.sz_ratio = 1.0/self.sz_tm        self.batch_size = batch_size        self.channal = channal        self.train = False    def build(self):        # 计算积分图，均值等        ##tm的均值和平方均值        self.tm_sum = tf.reshape(tf.reduce_sum(self.tensor_tm,axis=(1,2)),[self.batch_size,1,1,1])  #[batch,1,1,1]        self.tm_sum_sq = tf.reshape(tf.reduce_sum(self.tensor_tm * self.tensor_tm,axis=(1,2)),[self.batch_size,1,1,1]) #[batch,1,1,1]        ##ref的积分图和平方积分图        self.tensor_ref_sq = self.tensor_ref * self.tensor_ref        # 站位，在外面计算        self.ref_integral_img = tf.placeholder(dtype=tf.float32, shape=(self.batch_size,self.h_ref, self.w_ref,1))        self.ref_integral_img_sq = tf.placeholder(dtype=tf.float32, shape=(self.batch_size,self.h_ref, self.w_ref,1))        self.ref_div_sq_mat = tf.placeholder(dtype=tf.float32, shape=(self.batch_size,self.h_ref - self.h_tm + 1, self.w_ref - self.w_tm + 1,1))        ##模板去均值        self.tensor_tm = self.tensor_tm - self.tm_sum / self.sz_tm        ##拼成一个大的tensor        self.tensor_left = tf.zeros((self.batch_size,self.h_tm, self.w_ref - self.w_tm,1), dtype=tf.float32)        self.tensor_bottom = tf.zeros((self.batch_size,self.h_ref - self.h_tm, self.w_ref,1), dtype=tf.float32)        self.tensor_tm_ex = tf.concat([tf.concat([self.tensor_tm, self.tensor_left], axis=2), self.tensor_bottom], axis=1)        ###fft计算卷积        self.res = []        for b in range(self.batch_size):            self.F_tm = tf.fft2d(tf.complex(-self.tensor_tm_ex[b,:,:,0], tf.zeros([h_ref,w_ref])))            self.F_tm = -tf.conj(self.F_tm)            self.F_ref = tf.fft2d(tf.complex(self.tensor_ref[b,:,:,0], tf.zeros([h_ref,w_ref])))            # F_m = tf.multiply(F_ref,F_tm)            self.F_m = self.F_ref * self.F_tm            self.res.append(tf.real(tf.ifft2d(self.F_m))[0:self.h_ref - self.h_tm + 1, 0:self.w_ref - self.w_tm + 1])        self.res = tf.stack(self.res)        self.res = tf.reshape(self.res,[batch_size,h_ref-h_tm+1,w_ref-w_tm+1,1])        # 计算分母        self.tm_div = self.tm_sum_sq - self.sz_ratio * self.tm_sum * self.tm_sum        self.res = self.res / tf.sqrt(self.ref_div_sq_mat * self.tm_div)        ##训练        self.optmizer = tf.train.AdamOptimizer(0.05)        self.loss = np.sum(-self.res[:,h_tm, w_tm,0])        self.train_res = self.optmizer.minimize(self.loss)        return self.res    def run(self,sess):        ref_arr_sq = sess.run(self.tensor_ref_sq)        ref_arr = sess.run(self.tensor_ref)        in_ref_sq = calcIntag_batch(ref_arr_sq)        in_ref = calcIntag_batch(ref_arr)        ref_div_sq = calcDivmat_batch(in_ref_sq,in_ref,self.h_tm,self.w_tm)        if self.train:            res_out,train_out,loss_out = sess.run([self.res,self.train_res,self.loss], feed_dict={self.ref_integral_img: in_ref, self.ref_integral_img_sq: in_ref_sq,                                       self.ref_div_sq_mat: ref_div_sq})            return  res_out#, train_out, loss_out        else:            res_out = sess.run(self.res, feed_dict={self.ref_integral_img: in_ref, self.ref_integral_img_sq: in_ref_sq,                                                       self.ref_div_sq_mat: ref_div_sq})            return res_outif __name__ == "__main__":    import cv2    import numpy as np    import matplotlib.pyplot as plt    ## read img #####    tm_path_list = ['/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/2_AB_real_B.png',               '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/1_AB_real_B.png',                    '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/3_AB_real_B.png']    ref_path_list = ['/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/2_AB_real_B.png',                '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/1_AB_real_B.png',                     '/home/wdh/pytorch-CycleGAN-and-pix2pix1_run/results/map_pix2pix/test_latest/real_B/3_AB_real_B.png']    img_tm_gray = []    img_ref_gray = []    batch_size = len(tm_path_list)    i=0    for tm_path,ref_path in zip(tm_path_list,ref_path_list):        img_ref_bgr = cv2.imread(ref_path)        img_tm_bgr = cv2.imread(tm_path)        img_ref_gray.append((cv2.cvtColor(img_ref_bgr, cv2.COLOR_BGR2GRAY)) / 255.0)        img_src_gray = (cv2.cvtColor(img_tm_bgr, cv2.COLOR_BGR2GRAY)) / 255.0        if i == 0:            [h_ref, w_ref] = img_ref_gray[0].shape            [h_tm, w_tm] = img_src_gray.shape            h_tm = int(h_tm / 3)            w_tm = int(w_tm / 3)            sz_tm = h_tm * w_tm            sz_ratio = 1.0 / (sz_tm)            channal = 1        # img_tm_gray = np.zeros([h_ref,w_ref])        img_tm_gray.append(np.copy(img_src_gray[h_tm:h_tm + h_tm, w_tm:w_tm + w_tm]))        i = i + 1    img_tm_gray = np.array(img_tm_gray)    img_ref_gray = np.array(img_ref_gray)    tensor_tm_src = tf.reshape(tf.convert_to_tensor(img_tm_gray, dtype=tf.float32),[batch_size,h_tm,w_tm,1])  #[batch,h_ref,w_ref,channal]    tensor_ref_src = tf.reshape(tf.convert_to_tensor(img_ref_gray, dtype=tf.float32),[batch_size,h_ref,w_ref,1])    Weight_tm = tf.Variable(tf.ones([1,h_tm,w_tm,1]), dtype=tf.float32)    Weight_ref = tf.Variable(tf.ones([1,h_ref,w_ref,1]), dtype=tf.float32)    tensor_tm = Weight_tm * tensor_tm_src    tensor_ref = Weight_ref * tensor_ref_src    ncc = NCC_Layer(tensor_ref,tensor_tm,h_ref,w_ref,h_tm,w_tm,batch_size,channal)    res_node = ncc.build()    init = tf.global_variables_initializer()    with tf.Session() as sess:        sess.run(init)        ncc.train = True        for i in range(5):            tm_img_show = sess.run(ncc.tensor_tm)            ref_img_show = sess.run(ncc.tensor_ref)            res_out = ncc.run(sess)        plt.figure()        for b in range(batch_size):            ref_mat = np.float32(np.copy(img_ref_gray[b,:,:]))            tm_mat = np.float32(np.copy(img_tm_gray[b,:,:]))            res2 = cv2.matchTemplate(ref_mat,tm_mat , 5)            print(np.max(res_out), np.min(res_out),np.max(res2), np.min(res2))            plt.subplot(batch_size,5,b*5+1),plt.title('tm'),plt.imshow(tm_img_show[b,:,:,0],cmap='gray')            plt.subplot(batch_size,5,b*5+2),plt.title('ref'), plt.imshow(ref_img_show[b,:,:,0], cmap='gray')            plt.subplot(batch_size,5,b*5+3),plt.title('cv_corr_map'), plt.imshow(res2, cmap='gray')            plt.subplot(batch_size,5,b*5+4),plt.title('layer_corr_map'),plt.imshow(res_out[b,:,:,0],cmap='gray')            plt.subplot(batch_size,5,b*5+5),plt.title('diff_corr'), plt.imshow(res_out[b,:,:,0]-res2,cmap='gray',vmax=1.0,vmin=-1.0)        # plt.imshow(res_out-res2,cmap='gray',vmax=1.0,vmin=-1.0)        plt.show()