1
import numpy as np
2
from tqdm import tqdm, trange
3

4
room_num = 6
5
room_paths = [(0, 4), (3, 4), (3, 1), (1, 5), (2, 3), (4, 5)]
6
target_room = 5
7

8
# Q 矩阵，默认值填充0
9
Q = np.zeros((room_num, room_num))
10
# R 矩阵，默认值填充-1
11
reward = np.full((room_num, room_num), -1)
12
# 有路径的房间，奖励设为0
13
for room_path in room_paths:
14
    if room_path[1] == target_room:
15
        reward[room_path[0]][room_path[1]] = 100 # 房间到达目标房间，奖励设为100
16
    else:
17
        reward[room_path[0]][room_path[1]] = 0
18

19
    # 双向路径
20
    if room_path[0] == target_room:
21
        reward[room_path[1]][room_path[0]] = 100 # 房间到达目标房间，奖励设为100
22
    else:
23
        reward[room_path[1]][room_path[0]] = 0
24

25
reward[target_room][target_room] = 100 # 目标房间奖励设为100
26
print("reward:")
27
print(reward)
28

29
max_epoch = 2000
30
lamma = 0.8
31
modes = ['one-path', 'one-step']
32
# one-path 一直走，直到到达目标房间
33
# one-step 只走一步
34
mode = modes[1]
35

36
for epoch in trange(max_epoch):
37
    new_Q = Q.copy()
38
    current_state = np.random.randint(0, room_num)
39

40
    def one_step(current_state, Q, reward, lamma):
41
        # 随机选择一个可行的动作
42
        p_action = (reward[current_state] >= 0).astype(int) / np.sum(reward[current_state] >= 0)
43
        current_action = np.random.choice(room_num, p=p_action)
44
        # 更新 Q 矩阵
45
        new_Q[current_state][current_action] = reward[current_state][current_action] + lamma * np.max(Q[current_action])
46
        new_state = current_action
47
        return new_state, new_Q
48

49
    if mode == 'one-step':
50
        _, new_Q = one_step(current_state, Q, reward, lamma)
51
    else:
52
        while current_state != target_room:
53
            current_state, new_Q = one_step(current_state, Q, reward, lamma)
54
    Q = new_Q
55

56
print("Q:")
57
print(Q.round())