TensorFlow强化学习入门（1）——双臂赌博机

import tensorflow as tf
import numpy as np
# 定义一个四臂赌博机
# pullBandit函数会根据均值为0的正态分布来生成随机数，输入的bandit数越小，得到正收益的概率越大
# 我们的目标是agent可以一直选择带来正收益的赌博机
# 列出我们的赌博机bandit值，按照下面的设定，第四个机臂(index为3)带来正收益的可能性最大。
bandits = [0.2, 0, -0.2, -5]
num_bandits = len(bandits)
def pullBandit(bandit):
    # 获取随机数
    result = np.random.randn(1)
    if result > bandit:
        # 返回正收益
        return 1
    else:
        return -1

# 建立agent，包含了对每个赌博机的价值预估，使用策略梯度方法进行更新
tf.reset_default_graph()
# 建立网络的前馈部分,通过它可以实现决策
weights = tf.Variable(tf.ones([num_bandits]))
chosen_action = tf.argmax(weights, 0)
# 网络训练部分,通过计算损失值来更新网络
reward_holder = tf.placeholder(shape=[1], dtype=tf.float32)
action_holder = tf.placeholder(shape=[1], dtype=tf.int32)
responsible_weight = tf.slice(weights, action_holder, [1])
loss = -(tf.log(responsible_weight)*reward_holder)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
update = optimizer.minimize(loss)

# 训练agent
total_episodes = 1000
total_reward = np.zeros(num_bandits)
e = 0.1 # 随机决策概率
init = tf.initialize_all_variables()
# 启动TensorFlow计算图
with tf.Session() as sess:
    sess.run(init)
    i = 0
    while i < total_episodes:
        # 根据价值决策或随机决策
        if np.random.rand(1) < e:
            action = np.random.randint(num_bandits)
        else:
            action = sess.run(chosen_action)
        reward = pullBandit(bandits[action]) # 得到所选赌博机的收益
        # 更新网络
        _, resp, ww = sess.run([update, responsible_weight, weights], feed_dict={reward_holder:[reward], action_holder:[action]})
        # 更新决策分值
        total_reward[action] += reward
        if i % 50 == 0:
            print(str(num_bandits) + "个老虎机的实时收益为:" + str(total_reward))
        i += 1
print("agent认为" + str(np.argmax(ww)+1) + "号老虎机的收益最高")
if np.argmax(ww) == np.argmax(-np.array(bandits)):
    print("经验证正确!")
else:
    print("经验证错误!")

4个机臂的实时收益为:[1. 0. 0. 0.]
4个机臂的实时收益为:[-2. -1.  1. 31.]
4个机臂的实时收益为:[-2. -1.  2. 78.]
4个机臂的实时收益为:[ -1.   0.   2. 126.]
4个机臂的实时收益为:[ -1.   0.   3. 175.]
4个机臂的实时收益为:[  0.   0.   3. 222.]
4个机臂的实时收益为:[ -1.   0.   3. 269.]
4个机臂的实时收益为:[ -2.   0.   2. 315.]
4个机臂的实时收益为:[ -2.   0.   1. 364.]
4个机臂的实时收益为:[ -1.   3.   0. 409.]
4个机臂的实时收益为:[ -1.   3.   0. 453.]
4个机臂的实时收益为:[ -2.   1.   1. 499.]
4个机臂的实时收益为:[ -3.   0.   0. 544.]
4个机臂的实时收益为:[ -3.   1.   0. 591.]
4个机臂的实时收益为:[ -3.   0.  -1. 639.]
4个机臂的实时收益为:[ -2.  -1.   0. 684.]
4个机臂的实时收益为:[ -1.  -1.   0. 731.]
4个机臂的实时收益为:[ -1.  -1.   0. 779.]
4个机臂的实时收益为:[ -2.   2.   0. 823.]
4个机臂的实时收益为:[ -2.   1.   2. 870.]
agent认为4号机臂的收益最高
经验证正确!