遗传算法与进化策略

import gym
env = gym.make('Your-Env-Name') #初始化训练环境
observation = env.reset         #初始化一个事件（Eposide）
for _ in range(MAX_STEP):
    # 根据observation制定下一步行动
    ...
    # 执行行动获取结果
    observation, reward, done, info = env.step(YOUR_ACTION)
    # 根据reward修改个体适应度
    ...
    # 如果事件结束则退出循环
    if done:
        break

import numpy as np
import random as rand

class Agent:

    def __init__(self):
        # 随机生成一个个体，包含由两个四维向量组成的矩阵，分别代表ES的两种DNA
        self.gene = np.random.rand(2, 4) * 10
        # 初始化个体适应度为0
        self.fitness = 0

    def __mul__(self, other):
        # 重载乘法运算使其成为两个个体产生子代的运算
        kid = Agent()
        # 下面的代码是随机遗传变异的过程，我的实现不是很好
        gene_T = kid.gene.T
        tup_DNA = (self.gene[0], other.gene[0])
        DNA_P = np.vstack(tup_DNA)
        tup_MUT = (self.gene[1], other.gene[1])
        MUT_P = np.vstack(tup_MUT)
        for col in range(4):
            rnd = rand.randint(0, 1)
            gene_T[col] = np.array([DNA_P.T[col][rnd], MUT_P.T[col][rnd]])
            # 这里对生成的新基因和变异率加以符合正态分布N(0，σ^2 )的变异，其中σ为对应基因的变异率
            gene_T[col][0] += np.random.randn() * gene_T[col][1]
            gene_T[col][1] += np.random.randn() * gene_T[col][1]
        kid.gene = gene_T.T
        return kid

    def calc(self, observation):
        # 根据观测值决定行动
        res = np.sum(self.gene[0] * observation)
        if res > 0:
            return 1
        if res < 0:
            return 0
        return 0

agents = [Agent() for _ in range(population)]

import gym

def get_fittness(agents):
    # 在Gym中训练并获得个体适应度
    fitness = []
    for agent in agents:
        agent.fitness = 0
    for agent in agents:
        observation = env.reset()
        for _ in range(50000):
            observation, reward, done, info = env.step(agent.calc(observation))
            if done:
                break
            # 根据获得的奖励调整个体适应度
            agent.fitness += reward

        fitness.append(agent.fitness)
    return np.array(fitness)

def choose_agent(agents):
    # 获取每个个体的适应度
    fitness = get_fittness(agents)
    # 将是适应度从大到小排序
    fitness_sort = np.argsort(-fitness)
    # 选取好的个体加入新种群，淘汰不好的个体
    new_agents = []
    for i in range(keep):
        new_agents.append(agents[fitness_sort[i]])
    # 我这里顺便返回了表现最好的个体以供展示
    return new_agents, fitness, agents[fitness_sort[0]]

def make_agent(agents):
    # 从选择好的个体中随机选出两个产生新的个体，直到种群再次达到规定数目
    for _ in range(population - len(agents)):
        agents.append(agents[rand.randint(0, keep - 1)]
                      * agents[rand.randint(0, keep - 1)])
    return agents

import matplotlib.pyplot as plt

# 制定训练环境
env = gym.make('CartPole-v1')

# 规定参数并生成初始种群
keep = 60
population = 100
agents = [Agent() for _ in range(population)]

# 创建用于保存绘图数据的列表
fitness_avg = []
fitness_max = []
fitness_min = []

# 训练主循环
for i in range(100):
    agents = make_agent(agents)                  # 生成种群
    agents, fitness, best = choose_agent(agents) # 自然选择
    fitness_avg.append(np.mean(fitness))         # 保存这一代适应度数据
    fitness_max.append(np.max(fitness))
    fitness_min.append(np.min(fitness))
    print("%d Done! Avg: %d Max: %d" % (i, np.mean(fitness), np.max(fitness)))
    # 每训练10代，挑选出最好的个体展示其行为
    if i % 10 == 0:
        observation = env.reset()
        for _ in range(10000):
            # 将上一动作渲染到窗口
            env.render()
            observation, reward, done, info = env.step(
                best.calc(observation))
            if done:
                break

plt.figure()
plt.plot(fitness_avg)
plt.plot(fitness_max)
plt.plot(fitness_min)
plt.show()