A2C Advantage Actor-Critic源码

A2C Advantage Actor-Critic (离散空间)

标签（空格分隔）： 增强学习算法 源码

---

import numpy as npimport tensorflow as tfimport gymnp.random.seed(2)tf.set_random_seed(2)  # reproducible# SuperparametersOUTPUT_GRAPH = False # 是否保存模型（网络结构）MAX_EPISODE = 3000DISPLAY_REWARD_THRESHOLD = 200  # renders environment if total episode reward is greater then this thresholdMAX_EP_STEPS = 100000   # maximum time step in one episodeRENDER = True  # rendering wastes timeGAMMA = 0.9     # reward discount in TD errorLR_A = 0.1    # learning rate for actorLR_C = 0.1     # learning rate for criticenv = gym.make('MountainCar-v0')env.seed(1)  # reproducibleenv = env.unwrappedN_F = env.observation_space.shape[0]   #状态空间的维度N_A = env.action_space.n                #动作空间的维度class Actor(object):                    #Policy net    def __init__(self, sess, n_features, n_actions, lr=0.001):        self.sess = sess        self.s = tf.placeholder(tf.float32, [1, n_features], "state")   #state input        self.a = tf.placeholder(tf.int32, None, "act")                  #action input        self.td_error = tf.placeholder(tf.float32, None, "td_error")    # TD_error        with tf.variable_scope('Actor'):            l1 = tf.layers.dense(                inputs=self.s,                units=20,    # number of hidden units                activation=tf.nn.relu,                kernel_initializer=tf.random_normal_initializer(0., .1),    # weights                bias_initializer=tf.constant_initializer(0.1),  # biases                name='l1'            )                   #fully connected layer 1            self.acts_prob = tf.layers.dense(                inputs=l1,                units=n_actions,    # output units                activation=tf.nn.softmax,   # get action probabilities                kernel_initializer=tf.random_normal_initializer(0., .1),  # weights                bias_initializer=tf.constant_initializer(0.1),  # biases                name='acts_prob'            )               #output softmax        with tf.variable_scope('exp_v'):            log_prob = tf.log(self.acts_prob[0, self.a])        #selecte the action prob value            self.exp_v = tf.reduce_mean(log_prob * self.td_error)  # advantage (TD_error) guided loss        with tf.variable_scope('train'):            self.train_op = tf.train.AdamOptimizer(lr).minimize(-self.exp_v)  # minimize(-exp_v) = maximize(exp_v)    def learn(self, s, a, td):        s = s[np.newaxis, :]        feed_dict = {self.s: s, self.a: a, self.td_error: td}  #td temproal difference 由 critic net产生        _, exp_v = self.sess.run([self.train_op, self.exp_v], feed_dict)  #优化        return exp_v    def choose_action(self, s):        s = s[np.newaxis, :]        probs = self.sess.run(self.acts_prob, {self.s: s})   # get probabilities for all actions        return np.random.choice(np.arange(probs.shape[1]), p=probs.ravel())   # return a int 以某种概率选择动作class Critic(object):    def __init__(self, sess, n_features, lr=0.01):        self.sess = sess        self.s = tf.placeholder(tf.float32, [1, n_features], "state")   #critic net input 当前状态        self.v_ = tf.placeholder(tf.float32, [1, 1], "v_next")         #下一状态对应的value 值        self.r = tf.placeholder(tf.float32, None, 'r')                  #当前状态执行动作后的奖励值        with tf.variable_scope('Critic'):                           #构建critic 网络，注意输出一个值表示当前value            l1 = tf.layers.dense(                inputs=self.s,                units=20,  # number of hidden units                activation=tf.nn.relu,  # None                # have to be linear to make sure the convergence of actor.                # But linear approximator seems hardly learns the correct Q.                kernel_initializer=tf.random_normal_initializer(0., .1),  # weights                bias_initializer=tf.constant_initializer(0.1),  # biases                name='l1'            )            self.v = tf.layers.dense(                inputs=l1,                units=1,  # output units                activation=None,                kernel_initializer=tf.random_normal_initializer(0., .1),  # weights                bias_initializer=tf.constant_initializer(0.1),  # biases                name='V'            )        with tf.variable_scope('squared_TD_error'):            self.td_error = self.r + GAMMA * self.v_ - self.v   #贝尔慢迭代公式 求TD-error            self.loss = tf.square(self.td_error)    # TD_error = (r+gamma*V_next) - V_eval        with tf.variable_scope('train'):            self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)    def learn(self, s, r, s_):      #critic net的学习算法        s, s_ = s[np.newaxis, :], s_[np.newaxis, :]        v_ = self.sess.run(self.v, {self.s: s_})    #下一状态输入到网络获取下一状态的q值        td_error, _ = self.sess.run([self.td_error, self.train_op],                                              {self.s: s, self.v_: v_, self.r: r})  #当前状态下输入获取当前状态q值        return td_errorsess = tf.Session()actor = Actor(sess, n_features=N_F, n_actions=N_A, lr=LR_A)critic = Critic(sess, n_features=N_F, lr=LR_C)     # we need a good teacher, so the teacher should learn faster than the actorsess.run(tf.global_variables_initializer())if OUTPUT_GRAPH:    tf.summary.FileWriter("logs/", sess.graph)for i_episode in range(MAX_EPISODE):    s = env.reset()    t = 0    track_r = []    while True:        if RENDER: env.render()        a = actor.choose_action(s)        s_, r, done, info = env.step(a)        if done: r = -20        track_r.append(r)        td_error = critic.learn(s, r, s_)  # gradient = grad[r + gamma * V(s_) - V(s)]        actor.learn(s, a, td_error)     # true_gradient = grad[logPi(s,a) * td_error]        s = s_        t += 1        if done or t >= MAX_EP_STEPS:            ep_rs_sum = sum(track_r)            if 'running_reward' not in globals():                running_reward = ep_rs_sum            else:                running_reward = running_reward * 0.95 + ep_rs_sum * 0.05            if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True  # rendering            print("episode:", i_episode, "  reward:", int(running_reward))            break