对“视觉机器学习20讲配套仿真代码”的研究心得---增强学习

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%功能：演示增强学习算法在计算机视觉中的应用
%基于增强学习实现目标分类；
%环境：Win7，Matlab2012b
%Modi: NUDT-VAP
%时间：2014-02-04
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA


global3 % global parameters initialized 


stat=struct('Q',zeros(NS,NA),'iter',0,'old_action',1,'old_state',1,'current_state',1,'rimm',0,'total_reward',0);
done=0; % Pnemonic for simulation, 1 stands for end
        % 0 stands for continue 
while 0==done
    [stat,done]=jump_learn(stat);
end
   policy=pol_finder(stat);

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&

function action=action_selector(stat)


global NA 




ran=rand(1);


candidate=1;


sum=1/NA;


complete=0;


% Selecting each action with equal probability 


while 0==complete


        if ran<sum
        % action selected 
        action=candidate;
        complete=1;
        
        else          
        % test if ran is associated with next action 
        candidate=candidate+1;
        sum=sum+(1/NA);
        end
end

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&7

function [stat,done]=jump_learn(stat)


global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM


% This function simulates a jump and also updates the learning stats


old_state=stat.old_state;


old_action=stat.old_action;


% Determine current state 


current_state=state_finder(stat);


% Record Feedback in stat 


stat.current_state=current_state;


stat.rimm=TRM(old_state,current_state,old_action);


% DO LEARNING 


stat=qlearn(stat);


% Select next action 


next_action=action_selector(stat);


% Get ready to get out of this function 


stat.old_state=current_state;


stat.old_action=next_action;


if stat.iter>=ITERMAX

% Learning should end 


done=1;

else

done=0;
      
        end

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&7

function policy=pol_finder(stat)


global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM




for state=1:NS


[maxQfactor,index]=max(stat.Q(state,:));


policy(state)=index;


value_function(state)=maxQfactor;


end


policy


value_function


for state=1:NS


      for action=1:NA
      state
      action
      stat.Q(state,action)
      end


end

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&

function stat=qlearn(stat)


global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA


% Q-Learning 


% Finding the Max factor in the current state 


q_next=max(stat.Q(stat.current_state,:));




stat.iter=stat.iter+1;




%learn_rate=1/(stat.iter);


learn_rate=log(stat.iter+1)/(stat.iter+1);


%learn_rate=0.5*300/(300+stat.iter);


q=stat.Q(stat.old_state,stat.old_action);


q=q*(1-learn_rate)+(learn_rate*(stat.rimm+(LAMBDA*q_next)));


stat.Q(stat.old_state,stat.old_action)=q;

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&7

function candidate=state_finder(stat)


global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM


ran=rand(1);


old_action=stat.old_action;
old_state=stat.old_state;


sum=TPM(old_state,1,old_action);


candidate=1;


complete=0;


while 0==complete


        if ran<sum 
        
        complete=1;
        
        else
        
        candidate=candidate+1;


        sum=sum+TPM(old_state,candidate,old_action);


        end
        
end
                  

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&7

global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA




NO_REPLICATIONS=30; % No of replications of simulation 
ITERMAX=10000; % No of iterations of learning 
NA=2; % Number of actions in each state 
NS=2; % Number of states 


LAMBDA=0.8; % discount factor


SMALL=-1000000; 


TPM(:,:,1)=[0.7,0.3;0.4,0.6];


TPM(:,:,2)=[0.9,0.1;0.2,0.8];


TRM(:,:,1)=[6,-5;7,12];
    
TRM(:,:,2)=[10,17;-14,13];

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&

global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA




NO_REPLICATIONS=30; % No of replications of simulation 
ITERMAX=10000; % No of iterations of learning 
NA=2; % Number of actions in each state 
NS=2; % Number of states 


LAMBDA=0.8; % discount factor


SMALL=-1000000; 


TPM(:,:,1)=[0.7,0.3;0.4,0.6];


TPM(:,:,2)=[0.9,0.1;0.2,0.8];


TRM(:,:,1)=[6,5;7,12];
    
TRM(:,:,2)=[10,17;14,13];

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&

global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA




NO_REPLICATIONS=30; % No of replications of simulation 
ITERMAX=10000; % No of iterations of learning 
NA=2; % Number of actions in each state 
NS=2; % Number of states 


LAMBDA=0.8; % discount factor


SMALL=-1000000; 


TPM(:,:,1)=[0.7,0.3;0.4,0.6];


TPM(:,:,2)=[0.9,0.1;0.2,0.8];


TRM(:,:,1)=[6,-5;7,12];
    
TRM(:,:,2)=[12,17;-14,13];

global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA




NO_REPLICATIONS=30; % No of replications of simulation 
ITERMAX=10000; % No of iterations of learning 
NA=2; % Number of actions in each state 
NS=2; % Number of states 


LAMBDA=0.8; % discount factor


SMALL=-1000000; 


TPM(:,:,1)=[0.7,0.3;0.4,0.6];


TPM(:,:,2)=[0.9,0.1;0.2,0.8];


TRM(:,:,1)=[6,-5;7,12];
    
TRM(:,:,2)=[12,17;-14,13];

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&7

global NO_REPLICATIONS ITERMAX NA NS SMALL TPM TRM LAMBDA




NO_REPLICATIONS=30; % No of replications of simulation 
ITERMAX=10000; % No of iterations of learning 
NA=2; % Number of actions in each state 
NS=2; % Number of states 


LAMBDA=0.8; % discount factor


SMALL=-1000000; 


TPM(:,:,1)=[0.7,0.3;0.4,0.6];


TPM(:,:,2)=[0.9,0.1;0.2,0.8];


TRM(:,:,1)=[16,-5;7,12];
    
TRM(:,:,2)=[0,17;-14,13];

&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&7