ai_week.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ai_week.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ongEoQjaTJQk",
        "2qN_-pXpWncL",
        "Ce5w3VThXwpi",
        "ZDqmhWhRYLrJ",
        "kEcvEIxqHZLt",
        "G2__PdzbNrPi",
        "RBxwrlSzHZLu",
        "tO6bmhG1HZL0",
        "mCwoQwnGHZL5",
        "MtfJT8XbmsUo",
        "wNelbGV7HZME",
        "3Eqej3v50a91",
        "_8RT_QFdHZMH",
        "2DE9jeb_dpC0",
        "2Etj2gNUdpC2",
        "fh9kyyFKdpDT",
        "XWgEZaocdpDX",
        "fBwk9pOxdpDb",
        "sYiKfw4hdpDd",
        "z3RoVAbldpDf",
        "VFRbnqs1dpDi",
        "LLvt1QMmdpDm",
        "ny_CxfsodpDo",
        "xM4zo9nDdpDr",
        "vdXFMZdJdpDu"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dandanelbaz/ai_week/blob/master/ai_week.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ongEoQjaTJQk",
        "colab_type": "text"
      },
      "source": [
        "# Install coach\n",
        "Just use pip"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aY02_jIDcake",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pip install rl_coach"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "atlCxwoxVVxp"
      },
      "source": [
        "# AI Week Workshop "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ehMYL-BjVVxx"
      },
      "source": [
        "### ***1 Runing Coach*** "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "D3ecZ4H-VVxy"
      },
      "source": [
        "##### ***1.1 Training with default parameters*** "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "g3UYDJ2pVVxz",
        "colab": {}
      },
      "source": [
        "from rl_coach.agents.dqn_agent import DQNAgentParameters\n",
        "from rl_coach.environments.gym_environment import GymEnvironmentParameters, Atari, atari_schedule\n",
        "from rl_coach.graph_managers.graph_manager import VisualizationParameters\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "\n",
        "\n",
        "# creating graph manager\n",
        "graph_mgr = BasicRLGraphManager(\n",
        "    agent_params = DQNAgentParameters(), \n",
        "    env_params = Atari(level = 'Breakout-v0'), \n",
        "    schedule_params = atari_schedule, \n",
        "    vis_params = VisualizationParameters())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4HJwJOd5VVx3",
        "colab": {}
      },
      "source": [
        "graph_mgr.improve()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hgdMghauVVx6"
      },
      "source": [
        "##### ***1.2 Changing default parameters***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6Bq1hpqmVVx7",
        "colab": {}
      },
      "source": [
        "from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters\n",
        "from rl_coach.environments.gym_environment import GymVectorEnvironment\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import SimpleSchedule\n",
        "from rl_coach.architectures.embedder_parameters import InputEmbedderParameters\n",
        "\n",
        "# Reset tensorflow graph as the network has changed.\n",
        "import tensorflow as tf\n",
        "tf.reset_default_graph()\n",
        "\n",
        "# Define the environment parameters\n",
        "bit_length = 10\n",
        "env_params = GymVectorEnvironment(level='rl_coach.environments.toy_problems.bit_flip:BitFlip')\n",
        "env_params.additional_simulator_parameters = {'bit_length': bit_length, 'mean_zero': True}\n",
        "\n",
        "# Clipped PPO\n",
        "agent_params = ClippedPPOAgentParameters()\n",
        "agent_params.network_wrappers['main'].input_embedders_parameters = {\n",
        "    'state': InputEmbedderParameters(scheme=[]),\n",
        "    'desired_goal': InputEmbedderParameters(scheme=[])\n",
        "}\n",
        "\n",
        "graph_manager = BasicRLGraphManager(\n",
        "    agent_params=agent_params,\n",
        "    env_params=env_params,\n",
        "    schedule_params=SimpleSchedule()\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "LQWlNtIMVVx9",
        "colab": {}
      },
      "source": [
        "graph_manager.improve()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "1yyYBFROVVx_"
      },
      "source": [
        "##### ***1.3 Running a Coach preset***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "SGKOBh1IVVyA"
      },
      "source": [
        "When running Coach from the command line, we use a Preset module to define the experiment parameters. As its name implies, a preset is a predefined set of parameters to run some agent on some environment. Coach has many predefined presets that follow the algorithms definitions in the published papers, and allows training some of the existing algorithms with essentially no coding at all. This presets can easily be run from the command line. For example:\n",
        "\n",
        "**coach -p CartPole_DQN**\n",
        "\n",
        "You can find all the predefined presets under the presets directory, or by listing them using the following command:\n",
        "\n",
        "**coach -l**\n",
        "\n",
        "Coach can also be used with an externally defined preset by passing the absolute path to the module and the name of the graph manager object which is defined in the preset:\n",
        "\n",
        "**coach -p /home/my_user/my_agent_dir/my_preset.py:graph_manager**\n",
        "\n",
        "Some presets are generic for multiple environment levels, and therefore require defining the specific level through the command line:\n",
        "\n",
        "**coach -p Mujoco_ClippedPPO -lvl humanoid**\n",
        "\n",
        "There are plenty of other command line arguments you can use in order to customize the experiment. A full documentation of the available arguments can be found using the following command:\n",
        "\n",
        "**coach -h**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Hpj-e-0VVVyB",
        "colab": {}
      },
      "source": [
        "from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters\n",
        "from rl_coach.architectures.layers import Dense\n",
        "from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters, DistributedCoachSynchronizationType\n",
        "from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps\n",
        "from rl_coach.environments.environment import SingleLevelSelection\n",
        "from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2\n",
        "from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters\n",
        "from rl_coach.filters.filter import InputFilter\n",
        "from rl_coach.filters.observation.observation_normalization_filter import ObservationNormalizationFilter\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import ScheduleParameters\n",
        "from rl_coach.schedules import LinearSchedule\n",
        "\n",
        "####################\n",
        "# Graph Scheduling #\n",
        "####################\n",
        "\n",
        "schedule_params = ScheduleParameters()\n",
        "schedule_params.improve_steps = TrainingSteps(10000000)\n",
        "schedule_params.steps_between_evaluation_periods = EnvironmentSteps(2048)\n",
        "schedule_params.evaluation_steps = EnvironmentEpisodes(5)\n",
        "schedule_params.heatup_steps = EnvironmentSteps(0)\n",
        "\n",
        "#########\n",
        "# Agent #\n",
        "#########\n",
        "agent_params = ClippedPPOAgentParameters()\n",
        "\n",
        "\n",
        "agent_params.network_wrappers['main'].learning_rate = 0.0003\n",
        "agent_params.network_wrappers['main'].input_embedders_parameters['observation'].activation_function = 'tanh'\n",
        "agent_params.network_wrappers['main'].input_embedders_parameters['observation'].scheme = [Dense(64)]\n",
        "agent_params.network_wrappers['main'].middleware_parameters.scheme = [Dense(64)]\n",
        "agent_params.network_wrappers['main'].middleware_parameters.activation_function = 'tanh'\n",
        "agent_params.network_wrappers['main'].batch_size = 64\n",
        "agent_params.network_wrappers['main'].optimizer_epsilon = 1e-5\n",
        "agent_params.network_wrappers['main'].adam_optimizer_beta2 = 0.999\n",
        "\n",
        "agent_params.algorithm.clip_likelihood_ratio_using_epsilon = 0.2\n",
        "agent_params.algorithm.clipping_decay_schedule = LinearSchedule(1.0, 0, 1000000)\n",
        "agent_params.algorithm.beta_entropy = 0\n",
        "agent_params.algorithm.gae_lambda = 0.95\n",
        "agent_params.algorithm.discount = 0.99\n",
        "agent_params.algorithm.optimization_epochs = 10\n",
        "agent_params.algorithm.estimate_state_value_using_gae = True\n",
        "# Distributed Coach synchronization type.\n",
        "agent_params.algorithm.distributed_coach_synchronization_type = DistributedCoachSynchronizationType.SYNC\n",
        "\n",
        "agent_params.input_filter = InputFilter()\n",
        "agent_params.exploration = AdditiveNoiseParameters()\n",
        "agent_params.pre_network_filter = InputFilter()\n",
        "agent_params.pre_network_filter.add_observation_filter('observation', 'normalize_observation',\n",
        "                                                       ObservationNormalizationFilter(name='normalize_observation'))\n",
        "\n",
        "###############\n",
        "# Environment #\n",
        "###############\n",
        "env_params = GymVectorEnvironment(level=SingleLevelSelection(mujoco_v2))\n",
        "# Set the target success\n",
        "env_params.target_success_rate = 1.0\n",
        "\n",
        "graph_manager = BasicRLGraphManager(agent_params=agent_params, env_params=env_params,\n",
        "                                    schedule_params=schedule_params, vis_params=VisualizationParameters(),\n",
        "                                    preset_validation_params=preset_validation_params)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NGXXLZEfVVyD",
        "colab": {}
      },
      "source": [
        "!coach -l"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4kTtbkn0VVyF"
      },
      "source": [
        "### ***2 Adding a new environment***\n",
        "\n",
        "In this section we will implement the short corridor environment from Sutton & Barto Book.\n",
        "\n",
        "![short_corridor](https://drive.google.com/uc?id=1rYLI9dC92sfpF0BVxVENF964MfWJkxZq)\n",
        "\n",
        "*   Three non terminal states - The location of the agent\n",
        "\n",
        "*   The observations are one-hot encoding of the states\n",
        "*   Actions are reversed in the second state\n",
        "\n",
        "\n",
        "*   Reward is -1 for each time step\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "VQS_rc-qVVyG"
      },
      "source": [
        "##### ***2.1 Helper function*** \n",
        "The following code snippet contains some defines and an one-hot encoding helper function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "0c1ced2c-381d-46c3-fea8-7949b5714a71",
        "id": "R91zF3UiVVyG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile short_corridor_env_helpper.py\n",
        "import numpy as np\n",
        "\n",
        "LEFT = 0\n",
        "RIGHT = 1\n",
        "START_STATE = 0\n",
        "GOAL_STATE = 3\n",
        "NUM_STATES = 4\n",
        "REVERSE_STATE = 1\n",
        "\n",
        "def to_one_hot(state):\n",
        "    observation = np.zeros((NUM_STATES,))\n",
        "    observation[state] = 1\n",
        "    return observation"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting short_corridor_env_helpper.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "qwTYewWuVVyJ"
      },
      "source": [
        "##### ***2.2 Write short corridor environment*** \n",
        "Compete the following functions:\n",
        " function and the step function\n",
        "\n",
        "1.   is_done - will return a boolean . True only at termination state\n",
        "\n",
        "2.   reset - Resets environment to initial state\n",
        "3.   step - Returns the next observation, reward, and the boolean flag done\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "* **complete code**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "dfbb5c75-cc7b-4b13-a5cf-8d4d7f1c44cf",
        "id": "v4Kt5kB2VVyK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile short_corridor_env.py\n",
        "import numpy as np\n",
        "import gym\n",
        "from gym import spaces\n",
        "from  short_corridor_env_helpper import *\n",
        "\n",
        "\n",
        "class ShortCorridorEnv(gym.Env):\n",
        "\n",
        "    def __init__(self):\n",
        "        # Class constructor- Initializes class variables and sets initial state\n",
        "        self.observation_space = spaces.Box(0, 1, shape=(NUM_STATES,))\n",
        "        self.action_space = spaces.Discrete(2)\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        '''\n",
        "        Resets the environment to start state\n",
        "        '''\n",
        "        # An integer representing the state. Number between zero and three\n",
        "        self.current_state = ???\n",
        "        observation = to_one_hot(???)\n",
        "        return observation\n",
        "\n",
        "    def _is_done(self, current_state):\n",
        "        '''\n",
        "        return done a Boolean- True only if we reached the goal state\n",
        "        '''\n",
        "        ???\n",
        "        return done\n",
        "\n",
        "    def step(self, action):\n",
        "        '''\n",
        "        Returns the next observation, reward, and the boolean flag done\n",
        "        '''\n",
        "\n",
        "        if action ==LEFT:\n",
        "          step = -1\n",
        "        elif action == RIGHT:\n",
        "           ???\n",
        "\n",
        "        if self.current_state == REVERSE_STATE:\n",
        "        ### Replace step = -1 with step = 1 and vise versa\n",
        "            ???\n",
        "\n",
        "        self.current_state += step\n",
        "        self.current_state = max(0, self.current_state)\n",
        "\n",
        "        observation = to_one_hot(self.current_state)\n",
        "        reward = ???\n",
        "        done = self._is_done(self.current_state)\n",
        "\n",
        "        return observation, reward, done, {}\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting short_corridor_env.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "iXnQx6pWVVyL"
      },
      "source": [
        "##### ***2.3 Write preset to run existing agent on the new environment***\n",
        "*We will use the same preset from DQN example*.\n",
        "\n",
        "Since our environment is already using Gym API we are almost good to go.\n",
        "\n",
        "When selecting the environment parametes in the preset use **GymEnvironmentParameters** and pass the path of the environment source code using the level parameter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "1b8d431b-64bd-4488-b92a-5aeef7bd93be",
        "id": "VwhE2ENgVVyM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile short_corridor_dqn_preset.py\n",
        "from rl_coach.environments.gym_environment import GymEnvironmentParameters\n",
        "from rl_coach.filters.filter import NoInputFilter, NoOutputFilter\n",
        "from rl_coach.agents.dqn_agent import DQNAgentParameters\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import SimpleSchedule\n",
        "from rl_coach.memories.memory import MemoryGranularity\n",
        "\n",
        "\n",
        "####################\n",
        "# Graph Scheduling #\n",
        "####################\n",
        "schedule_params = SimpleSchedule()\n",
        "\n",
        "\n",
        "#########\n",
        "# Agent #\n",
        "#########\n",
        "agent_params = DQNAgentParameters()\n",
        "agent_params.input_filter = NoInputFilter()\n",
        "agent_params.output_filter = NoOutputFilter()\n",
        "# DQN params\n",
        "# ER size\n",
        "agent_params.memory.max_size = (MemoryGranularity.Transitions, 40000)\n",
        "\n",
        "\n",
        "###############\n",
        "# Environment #\n",
        "###############\n",
        "env_params = GymEnvironmentParameters(level='short_corridor_env:ShortCorridorEnv')\n",
        "\n",
        "\n",
        "#################\n",
        "# Graph Manager #\n",
        "#################\n",
        "graph_manager = BasicRLGraphManager(agent_params=agent_params,\n",
        "                                    env_params=env_params,\n",
        "                                    schedule_params=schedule_params)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting short_corridor_dqn_preset.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "dBQJYkncVVyO"
      },
      "source": [
        "##### ***2.4 Run new preset***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "P72bI9NIVVyP",
        "colab": {}
      },
      "source": [
        "!coach -p /content/short_corridor_dqn_preset.py:graph_manager\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "TszPPPJXVVyR"
      },
      "source": [
        "### ***3 Adding a new agent***\n",
        "Coach modularity makes adding an agent a clean and simple task.\n",
        "Typically consists of four parts:\n",
        "\n",
        "\n",
        "1.   Implement an agent's specific network head (and loss)\n",
        "2.   Implement exploration policy (optional)\n",
        "3.   Define new parameters class that extends `AgentParameters`\n",
        "4.   Implement a preset to run the agent on some environment\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mHPfkMXsVVyR"
      },
      "source": [
        "##### ***3.1 Write stochastic output layer***\n",
        "We use stochastic policy, meaning that we only produce the probability of going left and going right.\n",
        "This layer takes in the input from previous layer, the middleware, and outputs two numbers. \n",
        "\n",
        "![Probabilistic output](https://drive.google.com/uc?id=1hB_AsKUlxlu43sMkPAFfLaK6Z5sz1I-n)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "8778bce5-9e93-4a17-e990-13bedd45e0c3",
        "id": "wqkWJyB8VVyS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile probabilistic_layer.py\n",
        "import tensorflow as tf\n",
        "from rl_coach.architectures.tensorflow_components.layers import Dense\n",
        "\n",
        "class ProbabilisticLayer(object):\n",
        "    def __init__(self, input_layer, num_actions):\n",
        "        super().__init__()\n",
        "        scores = Dense(num_actions)(input_layer, name='logit')\n",
        "        self.event_probs = tf.nn.softmax(scores, name=\"policy\")\n",
        "        # define the distributions for the policy and the old policy\n",
        "        self.policy_distribution = tf.contrib.distributions.Categorical(probs=self.event_probs)\n",
        "\n",
        "    def log_prob(self, action):\n",
        "        return self.policy_distribution.log_prob(action)\n",
        "\n",
        "    def layer_output(self):\n",
        "        return self.event_probs"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting probabilistic_layer.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "chY6h-6VVVyT"
      },
      "source": [
        "##### ***3.2 Implement network head i.e. implement the loss***\n",
        "The Head needs to inherit from the base class `Head`.\n",
        "\n",
        "In order to maximize the sum of rewards, we want to go in the following direction $-\\Sigma_i R_i \\nabla_Wlog(\\pi(a_i|x_i))$\n",
        "\n",
        "`Complete code`\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "cff705c5-5a5a-4f84-e461-542f233d79fb",
        "id": "mtvvJMi9VVyU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile simple_pg_head.py\n",
        "import tensorflow as tf\n",
        "from rl_coach.architectures.tensorflow_components.heads.head import Head\n",
        "from rl_coach.base_parameters import AgentParameters\n",
        "from rl_coach.spaces import SpacesDefinition\n",
        "from probabilistic_layer import ProbabilisticLayer\n",
        "\n",
        "\n",
        "class SimplePgHead(Head):\n",
        "    def __init__(self, agent_parameters: AgentParameters,\n",
        "                 spaces: SpacesDefinition, network_name: str,\n",
        "                 head_idx: int = 0, is_local: bool = True):\n",
        "        super().__init__(agent_parameters, spaces, network_name)\n",
        "\n",
        "        self.exploration_policy = agent_parameters.exploration\n",
        "\n",
        "    def _build_module(self, input_layer):\n",
        "        # Define inputs\n",
        "        actions = tf.placeholder(tf.int32, [None], name=\"actions\")\n",
        "        returns = tf.placeholder(tf.float32, [None], name=\"returns\")\n",
        "\n",
        "        # Two actions, left or right\n",
        "        policy_distribution = ProbabilisticLayer(input_layer, num_actions=2)\n",
        "\n",
        "        # calculate loss\n",
        "        log_prob = policy_distribution.log_prob(???)\n",
        "        # We only want to encourage good actions, so we multiply the log probability with ...\n",
        "        modulated_log_prob = ???\n",
        "        expected_modulated_log_prob = tf.reduce_mean(modulated_log_prob)\n",
        "\n",
        "        ### Coach bookeeping\n",
        "        # List of placeholders for additional inputs to the stochastic head \n",
        "        #(except from the middleware input)\n",
        "        self.input.append(actions)\n",
        "        # The output of the stochastic head, which is also the output of the network.\n",
        "        self.output.append(???)\n",
        "        # Placeholder for the target that we will use to train the network\n",
        "        self.target = returns\n",
        "        # The loss that we will use to train the network.\n",
        "        # We take the gradient of this loss and move in the opposite direction\n",
        "        self.loss = ???\n",
        "        tf.losses.add_loss(self.loss)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting simple_pg_head.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I6DTDUNDVVyW"
      },
      "source": [
        "##### ***3.3 Define exploration policy*** \n",
        "Every iteration we want to sample from the network output distribution i.e. toss a bias coin to get the agent's actual move\n",
        "\n",
        "**`Complete code`**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "3877ad93-6388-492e-ab05-b38a67362bb4",
        "id": "Uw-83GrDVVyX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile simple_pg_exploration.py\n",
        "\n",
        "import numpy as np\n",
        "from rl_coach.exploration_policies.exploration_policy import ExplorationPolicy, ExplorationParameters\n",
        "from rl_coach.spaces import ActionSpace\n",
        "\n",
        "\n",
        "class DiscreteExplorationParameters(ExplorationParameters):\n",
        "    @property\n",
        "    def path(self):\n",
        "        return 'simple_pg_exploration:DiscreteExploration'\n",
        "\n",
        "\n",
        "class DiscreteExploration(ExplorationPolicy):\n",
        "    \"\"\"\n",
        "    Discrete exploration policy is intended for discrete action spaces. It expects the action values to\n",
        "    represent a probability distribution over the action\n",
        "    \"\"\"\n",
        "    def __init__(self, action_space: ActionSpace):\n",
        "        \"\"\"\n",
        "        :param action_space: the action space used by the environment\n",
        "        \"\"\"\n",
        "        super().__init__(action_space)\n",
        "\n",
        "    def get_action(self, probabilities):\n",
        "        # choose actions according to the probabilities\n",
        "        chosen_action = np.random.choice(self.action_space.actions, p=???)\n",
        "        return chosen_action, probabilities\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting simple_pg_exploration.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "uITIKI9qVVyY"
      },
      "source": [
        "##### ***3.4 Define new agent parameters***\n",
        "Coach is modular!\n",
        "\n",
        "Each class in Coach has a complementary parameters class which defines its constructor. \n",
        "This is also true for the agent. The agent has a complementary `AgentParameters` class. This class enables selecting the paramenters of the agent sub modules.\n",
        "\n",
        "It consists of the following four parts:\n",
        "\n",
        "\n",
        "\n",
        "1.   Algorithm\n",
        "2.   Exploration\n",
        "3.   Memory\n",
        "4.   Networks\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sQPPE6T_VVyZ",
        "colab": {}
      },
      "source": [
        "%%writefile simple_pg_params.py\n",
        "from rl_coach.architectures.embedder_parameters import InputEmbedderParameters\n",
        "from rl_coach.architectures.head_parameters import HeadParameters\n",
        "from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters\n",
        "from rl_coach.base_parameters import NetworkParameters, AlgorithmParameters, \\\n",
        "    AgentParameters\n",
        "\n",
        "from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters\n",
        "from rl_coach.exploration_policies.categorical import CategoricalParameters\n",
        "from rl_coach.memories.episodic.single_episode_buffer import SingleEpisodeBufferParameters\n",
        "from rl_coach.spaces import DiscreteActionSpace, BoxActionSpace\n",
        "from rl_coach.agents.policy_optimization_agent import PolicyGradientRescaler\n",
        "from simple_pg_exploration import DiscreteExplorationParameters\n",
        "\n",
        "class SimplePgAgentParameters(AgentParameters):\n",
        "    def __init__(self):\n",
        "        super().__init__(algorithm=SimplePGAlgorithmParameters(),\n",
        "                         exploration=DiscreteExplorationParameters(),\n",
        "                         memory=SingleEpisodeBufferParameters(),\n",
        "                         networks={\"main\": SimplePgTopology()})\n",
        "    @property\n",
        "    def path(self):\n",
        "        #return 'simple_pg_agent:SimplePgAgent'\n",
        "        return 'rl_coach.agents.policy_gradients_agent:PolicyGradientsAgent'\n",
        "\n",
        "        \n",
        "    \n",
        "# Since we are adding a new head we need to tell coach the heads path\n",
        "class SimplePgHeadParams(HeadParameters):\n",
        "    def __init__(self):\n",
        "        super().__init__(parameterized_class_name=\"AiWeekHead\")\n",
        "\n",
        "    @property\n",
        "    def path(self):\n",
        "        return 'simple_pg_head:SimplePgHead'\n",
        "\n",
        "\n",
        "class SimplePgTopology(NetworkParameters):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.input_embedders_parameters = {'observation': InputEmbedderParameters()}\n",
        "        self.middleware_parameters = FCMiddlewareParameters()\n",
        "        self.heads_parameters = [SimplePgHeadParams()]\n",
        "\n",
        "\n",
        "class SimplePGAlgorithmParameters(AlgorithmParameters):\n",
        "    \"\"\"\n",
        "    :param num_steps_between_gradient_updates: (int)\n",
        "        The number of steps between calculating gradients for the collected data. In the A3C paper, this parameter is\n",
        "        called t_max. Since this algorithm is on-policy, only the steps collected between each two gradient calculations\n",
        "        are used in the batch.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        # TOTAL_RETURN\n",
        "        # FUTURE_RETURN\n",
        "        # FUTURE_RETURN_NORMALIZED_BY_EPISODE \n",
        "        # FUTURE_RETURN_NORMALIZED_BY_TIMESTEP\n",
        "        # Q_VALUE\n",
        "        # A_VALUE\n",
        "        # TD_RESIDUAL\n",
        "        # DISCOUNTED_TD_RESIDUAL\n",
        "        # GAE\n",
        "        self.policy_gradient_rescaler = PolicyGradientRescaler.FUTURE_RETURN\n",
        "        self.num_steps_between_gradient_updates = 20000  # this is called t_max in all the papers\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "DX7UYlOsVVya"
      },
      "source": [
        "##### ***3.5 Write preset to run new agent on short corridor***\n",
        "complete code\n",
        "* **complete code**\n",
        "* **Hint: look at DQN preset**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "77fe8a73-efec-40af-86b2-7e2942ba2cc6",
        "id": "E6uDPM5HVVyb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile short_corridor_new_agent_preset.py\n",
        "from rl_coach.base_parameters import VisualizationParameters\n",
        "from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps\n",
        "from rl_coach.environments.gym_environment import GymEnvironmentParameters\n",
        "from rl_coach.filters.filter import NoInputFilter, NoOutputFilter\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import SimpleSchedule\n",
        "from rl_coach.memories.memory import MemoryGranularity\n",
        "from rl_coach.schedules import LinearSchedule\n",
        "from simple_pg_params import SimplePgAgentParameters\n",
        "\n",
        "\n",
        "####################\n",
        "# Graph Scheduling #\n",
        "####################\n",
        "schedule_params = SimpleSchedule()\n",
        "\n",
        "\n",
        "#########\n",
        "# Agent #\n",
        "#########\n",
        "agent_params = SimplePgAgentParameters()\n",
        "agent_params.input_filter = NoInputFilter()\n",
        "agent_params.output_filter = NoOutputFilter()\n",
        "\n",
        "\n",
        "###############\n",
        "# Environment #\n",
        "###############\n",
        "env_params = GymEnvironmentParameters(level='short_corridor_env:ShortCorridorEnv')\n",
        "\n",
        "#################\n",
        "# Graph Manager #\n",
        "#################\n",
        "graph_manager = BasicRLGraphManager(agent_params=agent_params,\n",
        "                                    env_params=env_params,\n",
        "                                    schedule_params=schedule_params)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing short_corridor_new_agent_preset.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "R5ouXmL6VVyd"
      },
      "source": [
        "##### ***3.6 Run preset of the new agent on the new environment***\n",
        "\n",
        "**`Complete code`**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "8f634c0d-4162-431c-a05d-ab7b7e0e1f37",
        "id": "ejBpXTBeVVyd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!coach -p /content/short_corridor_new_agent_preset.py:graph_manager"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
            "\u001b[30;46mPlease enter an experiment name: \u001b[0m\n",
            "\u001b]2;\u0007\n",
            "\u001b[30;46mCreating graph - name: BasicRLGraphManager\u001b[0m\n",
            "\u001b[30;46mCreating agent - name: agent\u001b[0m\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py:71: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
            "\n",
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/architecture.py:102: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py:240: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py:241: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py:242: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/layers.py:182: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dense instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/layers/python/layers/layers.py:1634: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.flatten instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py:313: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
            "\n",
            "WARNING:tensorflow:From ./probabilistic_layer.py:10: Categorical.__init__ (from tensorflow.python.ops.distributions.categorical) is deprecated and will be removed after 2019-01-01.\n",
            "Instructions for updating:\n",
            "The TensorFlow Distributions library has moved to TensorFlow Probability (https://github.com/tensorflow/probability). You should update all references to use `tfp.distributions` instead of `tf.distributions`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/distributions/categorical.py:242: Distribution.__init__ (from tensorflow.python.ops.distributions.distribution) is deprecated and will be removed after 2019-01-01.\n",
            "Instructions for updating:\n",
            "The TensorFlow Distributions library has moved to TensorFlow Probability (https://github.com/tensorflow/probability). You should update all references to use `tfp.distributions` instead of `tf.distributions`.\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/coach\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/coach.py\", line 777, in main\n",
            "    launcher.launch()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/coach.py\", line 226, in launch\n",
            "    self.run_graph_manager(graph_manager, args)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/coach.py\", line 612, in run_graph_manager\n",
            "    self.start_single_threaded(task_parameters, graph_manager, args)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/coach.py\", line 674, in start_single_threaded\n",
            "    start_graph(graph_manager=graph_manager, task_parameters=task_parameters)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/coach.py\", line 80, in start_graph\n",
            "    graph_manager.create_graph(task_parameters)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/graph_managers/graph_manager.py\", line 148, in create_graph\n",
            "    self.level_managers, self.environments = self._create_graph(task_parameters)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/graph_managers/basic_rl_graph_manager.py\", line 72, in _create_graph\n",
            "    level_manager = LevelManager(agents=agent, environment=env, name=\"main_level\")\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/level_manager.py\", line 91, in __init__\n",
            "    self.build(spaces_definition)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/level_manager.py\", line 165, in build\n",
            "    [agent.set_environment_parameters(spaces) for agent in self.agents.values()]\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/level_manager.py\", line 165, in <listcomp>\n",
            "    [agent.set_environment_parameters(spaces) for agent in self.agents.values()]\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/agents/agent.py\", line 335, in set_environment_parameters\n",
            "    self.init_environment_dependent_modules()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/agents/agent.py\", line 380, in init_environment_dependent_modules\n",
            "    self.networks = self.create_networks()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/agents/agent.py\", line 353, in create_networks\n",
            "    worker_device=self.worker_device)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/network_wrapper.py\", line 86, in __init__\n",
            "    network_is_trainable=True)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py\", line 74, in construct\n",
            "    return construct_on_device()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py\", line 59, in construct_on_device\n",
            "    return GeneralTensorFlowNetwork(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py\", line 126, in __init__\n",
            "    network_is_local, network_is_trainable)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/architecture.py\", line 105, in __init__\n",
            "    self.weights = self.get_model()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/general_network.py\", line 330, in get_model\n",
            "    self.output_heads[-1](head_input)\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/architectures/tensorflow_components/heads/head.py\", line 77, in __call__\n",
            "    self._build_module(squeeze_tensor(input_layer))\n",
            "  File \"./simple_pg_head.py\", line 33, in _build_module\n",
            "    self.input.append(actios)\n",
            "NameError: name 'actios' is not defined\n",
            "\n",
            "--------------------------------\n",
            "\n",
            "\u001b[30;46mResults stored at: ./experiments/16_11_2019-12_54\u001b[0m\n",
            "\u001b[30;46mTotal runtime: 0:00:09.115108\u001b[0m\n",
            "\n",
            "--------------------------------\n",
            "\n",
            "\u001b[30;46mDo you want to discard the experiment results (Warning: this cannot be undone)?\u001b[0m (y/N)Error in atexit._run_exitfuncs:\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/logger.py\", line 379, in summarize_experiment\n",
            "    if screen.ask_yes_no(\"Do you want to discard the experiment results (Warning: this cannot be undone)?\", False):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/rl_coach/logger.py\", line 154, in ask_yes_no\n",
            "    answer = input(\"{}{}{} ({})\".format(self._prefix_ask, title, self._suffix, default_answer))\n",
            "KeyboardInterrupt\n",
            "Error in atexit._run_exitfuncs:\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.6/multiprocessing/util.py\", line 262, in _run_finalizers\n",
            "    finalizer()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/util.py\", line 186, in __call__\n",
            "    res = self._callback(*self._args, **self._kwargs)\n",
            "  File \"/usr/lib/python3.6/multiprocessing/managers.py\", line 615, in _finalize_manager\n",
            "    process.join(timeout=1.0)\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 124, in join\n",
            "    res = self._popen.wait(timeout)\n",
            "  File \"/usr/lib/python3.6/multiprocessing/popen_fork.py\", line 46, in wait\n",
            "    from multiprocessing.connection import wait\n",
            "  File \"<frozen importlib._bootstrap>\", line 1007, in _handle_fromlist\n",
            "KeyboardInterrupt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "9qn7Qf9gHZLk"
      },
      "source": [
        "# AI Week Workshop solution"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5omK7SgIDAs2",
        "colab_type": "text"
      },
      "source": [
        "### ***1 Runing Coach*** "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2qN_-pXpWncL",
        "colab_type": "text"
      },
      "source": [
        "##### ***1.1 Training with default parameters*** "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sOhNdcRdW88Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from rl_coach.agents.dqn_agent import DQNAgentParameters\n",
        "from rl_coach.environments.gym_environment import GymEnvironmentParameters, Atari, atari_schedule\n",
        "from rl_coach.graph_managers.graph_manager import VisualizationParameters\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "\n",
        "\n",
        "# creating graph manager\n",
        "graph_mgr = BasicRLGraphManager(\n",
        "    agent_params = DQNAgentParameters(), \n",
        "    env_params = Atari(level = 'Breakout-v0'), \n",
        "    schedule_params = atari_schedule, \n",
        "    vis_params = VisualizationParameters())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wOE4FJpcW9L0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "graph_mgr.improve()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Ce5w3VThXwpi"
      },
      "source": [
        "##### ***1.2 Changing default parameters***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRs_jOghX1qN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters\n",
        "from rl_coach.environments.gym_environment import GymVectorEnvironment\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import SimpleSchedule\n",
        "from rl_coach.architectures.embedder_parameters import InputEmbedderParameters\n",
        "\n",
        "# Reset tensorflow graph as the network has changed.\n",
        "import tensorflow as tf\n",
        "tf.reset_default_graph()\n",
        "\n",
        "# Define the environment parameters\n",
        "bit_length = 10\n",
        "env_params = GymVectorEnvironment(level='rl_coach.environments.toy_problems.bit_flip:BitFlip')\n",
        "env_params.additional_simulator_parameters = {'bit_length': bit_length, 'mean_zero': True}\n",
        "\n",
        "# Clipped PPO\n",
        "agent_params = ClippedPPOAgentParameters()\n",
        "agent_params.network_wrappers['main'].input_embedders_parameters = {\n",
        "    'state': InputEmbedderParameters(scheme=[]),\n",
        "    'desired_goal': InputEmbedderParameters(scheme=[])\n",
        "}\n",
        "\n",
        "graph_manager = BasicRLGraphManager(\n",
        "    agent_params=agent_params,\n",
        "    env_params=env_params,\n",
        "    schedule_params=SimpleSchedule()\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I6K-bfr8X2va",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "graph_manager.improve()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ZDqmhWhRYLrJ"
      },
      "source": [
        "##### ***1.3 Running a Coach preset***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UcDkh_kVZJw7",
        "colab_type": "text"
      },
      "source": [
        "When running Coach from the command line, we use a Preset module to define the experiment parameters. As its name implies, a preset is a predefined set of parameters to run some agent on some environment. Coach has many predefined presets that follow the algorithms definitions in the published papers, and allows training some of the existing algorithms with essentially no coding at all. This presets can easily be run from the command line. For example:\n",
        "\n",
        "**coach -p CartPole_DQN**\n",
        "\n",
        "You can find all the predefined presets under the presets directory, or by listing them using the following command:\n",
        "\n",
        "**coach -l**\n",
        "\n",
        "Coach can also be used with an externally defined preset by passing the absolute path to the module and the name of the graph manager object which is defined in the preset:\n",
        "\n",
        "**coach -p /home/my_user/my_agent_dir/my_preset.py:graph_manager**\n",
        "\n",
        "Some presets are generic for multiple environment levels, and therefore require defining the specific level through the command line:\n",
        "\n",
        "**coach -p Mujoco_ClippedPPO -lvl humanoid**\n",
        "\n",
        "There are plenty of other command line arguments you can use in order to customize the experiment. A full documentation of the available arguments can be found using the following command:\n",
        "\n",
        "**coach -h**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0PM0h4NDYmrz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters\n",
        "from rl_coach.architectures.layers import Dense\n",
        "from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters, DistributedCoachSynchronizationType\n",
        "from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps\n",
        "from rl_coach.environments.environment import SingleLevelSelection\n",
        "from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2\n",
        "from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters\n",
        "from rl_coach.filters.filter import InputFilter\n",
        "from rl_coach.filters.observation.observation_normalization_filter import ObservationNormalizationFilter\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import ScheduleParameters\n",
        "from rl_coach.schedules import LinearSchedule\n",
        "\n",
        "####################\n",
        "# Graph Scheduling #\n",
        "####################\n",
        "\n",
        "schedule_params = ScheduleParameters()\n",
        "schedule_params.improve_steps = TrainingSteps(10000000)\n",
        "schedule_params.steps_between_evaluation_periods = EnvironmentSteps(2048)\n",
        "schedule_params.evaluation_steps = EnvironmentEpisodes(5)\n",
        "schedule_params.heatup_steps = EnvironmentSteps(0)\n",
        "\n",
        "#########\n",
        "# Agent #\n",
        "#########\n",
        "agent_params = ClippedPPOAgentParameters()\n",
        "\n",
        "\n",
        "agent_params.network_wrappers['main'].learning_rate = 0.0003\n",
        "agent_params.network_wrappers['main'].input_embedders_parameters['observation'].activation_function = 'tanh'\n",
        "agent_params.network_wrappers['main'].input_embedders_parameters['observation'].scheme = [Dense(64)]\n",
        "agent_params.network_wrappers['main'].middleware_parameters.scheme = [Dense(64)]\n",
        "agent_params.network_wrappers['main'].middleware_parameters.activation_function = 'tanh'\n",
        "agent_params.network_wrappers['main'].batch_size = 64\n",
        "agent_params.network_wrappers['main'].optimizer_epsilon = 1e-5\n",
        "agent_params.network_wrappers['main'].adam_optimizer_beta2 = 0.999\n",
        "\n",
        "agent_params.algorithm.clip_likelihood_ratio_using_epsilon = 0.2\n",
        "agent_params.algorithm.clipping_decay_schedule = LinearSchedule(1.0, 0, 1000000)\n",
        "agent_params.algorithm.beta_entropy = 0\n",
        "agent_params.algorithm.gae_lambda = 0.95\n",
        "agent_params.algorithm.discount = 0.99\n",
        "agent_params.algorithm.optimization_epochs = 10\n",
        "agent_params.algorithm.estimate_state_value_using_gae = True\n",
        "# Distributed Coach synchronization type.\n",
        "agent_params.algorithm.distributed_coach_synchronization_type = DistributedCoachSynchronizationType.SYNC\n",
        "\n",
        "agent_params.input_filter = InputFilter()\n",
        "agent_params.exploration = AdditiveNoiseParameters()\n",
        "agent_params.pre_network_filter = InputFilter()\n",
        "agent_params.pre_network_filter.add_observation_filter('observation', 'normalize_observation',\n",
        "                                                       ObservationNormalizationFilter(name='normalize_observation'))\n",
        "\n",
        "###############\n",
        "# Environment #\n",
        "###############\n",
        "env_params = GymVectorEnvironment(level=SingleLevelSelection(mujoco_v2))\n",
        "# Set the target success\n",
        "env_params.target_success_rate = 1.0\n",
        "\n",
        "graph_manager = BasicRLGraphManager(agent_params=agent_params, env_params=env_params,\n",
        "                                    schedule_params=schedule_params, vis_params=VisualizationParameters(),\n",
        "                                    preset_validation_params=preset_validation_params)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRZgLzdVYmue",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!coach -l"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "kEcvEIxqHZLt"
      },
      "source": [
        "### ***2 Adding a new environment***\n",
        "\n",
        "In this section we will implement the short corridor environment from Sutton & Barto Book.\n",
        "\n",
        "![short_corridor](https://drive.google.com/uc?id=1rYLI9dC92sfpF0BVxVENF964MfWJkxZq)\n",
        "\n",
        "*   Three non terminal states - The location of the agent\n",
        "\n",
        "*   The observations are one-hot encoding of the states\n",
        "*   Actions are reversed in the second state\n",
        "\n",
        "\n",
        "*   Reward is -1 for each time step\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G2__PdzbNrPi",
        "colab_type": "text"
      },
      "source": [
        "##### ***2.1 Helper function*** \n",
        "The following code snippet contains some defines and an one-hot encoding helper function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lnMACvXo_y3U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile short_corridor_env_helpper.py\n",
        "import numpy as np\n",
        "\n",
        "LEFT = 0\n",
        "RIGHT = 1\n",
        "START_STATE = 0\n",
        "GOAL_STATE = 3\n",
        "NUM_STATES = 4\n",
        "REVERSE_STATE = 1\n",
        "\n",
        "def to_one_hot(state):\n",
        "    observation = np.zeros((NUM_STATES,))\n",
        "    observation[state] = 1\n",
        "    return observation"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RBxwrlSzHZLu"
      },
      "source": [
        "##### ***2.2 Write short corridor environment*** \n",
        "Compete the following functions:\n",
        " function and the step function\n",
        "\n",
        "1.   is_done - will return a boolean . True only at termination state\n",
        "\n",
        "2.   reset - Resets environment to initial state\n",
        "3.   step - Returns the next observation, reward, and the boolean flag done\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "* **complete code**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uxwi5S1vHZLw",
        "colab": {}
      },
      "source": [
        "%%writefile short_corridor_env.py\n",
        "import numpy as np\n",
        "import gym\n",
        "from gym import spaces\n",
        "from  short_corridor_env_helpper import *\n",
        "\n",
        "\n",
        "class ShortCorridorEnv(gym.Env):\n",
        "\n",
        "    def __init__(self):\n",
        "        # Class constructor- Initializes class variables and sets initial state\n",
        "        self.observation_space = spaces.Box(0, 1, shape=(NUM_STATES,))\n",
        "        self.action_space = spaces.Discrete(2)\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        '''\n",
        "        Resets the environment to start state\n",
        "        '''\n",
        "        # An integer representing the state. Number between zero and three\n",
        "        self.current_state = START_STATE\n",
        "        observation = to_one_hot(self.current_state)\n",
        "        return observation\n",
        "\n",
        "    def _is_done(self, current_state):\n",
        "        '''\n",
        "        return done a Boolean- True only if we reached the goal state\n",
        "        '''\n",
        "        done = self.current_state == GOAL_STATE\n",
        "        return done\n",
        "\n",
        "    def step(self, action):\n",
        "        '''\n",
        "        Returns the next observation, reward, and the boolean flag done\n",
        "        '''\n",
        "\n",
        "        if action ==LEFT:\n",
        "          step = -1\n",
        "        elif action == RIGHT:\n",
        "           step = 1\n",
        "\n",
        "        if self.current_state == REVERSE_STATE:\n",
        "        ### Replace step = -1 with step = 1 and vise versa\n",
        "            step = -step\n",
        "\n",
        "        self.current_state += step\n",
        "        self.current_state = max(0, self.current_state)\n",
        "\n",
        "        observation = to_one_hot(self.current_state)\n",
        "        reward = -1\n",
        "        done = self._is_done(self.current_state)\n",
        "\n",
        "        return observation, reward, done, {}\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "tO6bmhG1HZL0"
      },
      "source": [
        "##### ***2.3 Write preset to run existing agent on the new environment***\n",
        "*We will use the same preset from DQN example*.\n",
        "\n",
        "Since our environment is already using Gym API we are almost good to go.\n",
        "\n",
        "When selecting the environment parametes in the preset use **GymEnvironmentParameters** and pass the path of the environment source code using the level parameter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wVmJ2CfbHZL1",
        "outputId": "3eb4dd2f-03f1-4bf0-bab6-373b2237cf17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile short_corridor_dqn_preset.py\n",
        "from rl_coach.environments.gym_environment import GymEnvironmentParameters\n",
        "from rl_coach.filters.filter import NoInputFilter, NoOutputFilter\n",
        "from rl_coach.agents.dqn_agent import DQNAgentParameters\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import SimpleSchedule\n",
        "from rl_coach.memories.memory import MemoryGranularity\n",
        "\n",
        "\n",
        "####################\n",
        "# Graph Scheduling #\n",
        "####################\n",
        "schedule_params = SimpleSchedule()\n",
        "\n",
        "\n",
        "#########\n",
        "# Agent #\n",
        "#########\n",
        "agent_params = DQNAgentParameters()\n",
        "agent_params.input_filter = NoInputFilter()\n",
        "agent_params.output_filter = NoOutputFilter()\n",
        "# DQN params\n",
        "# ER size\n",
        "agent_params.memory.max_size = (MemoryGranularity.Transitions, 40000)\n",
        "\n",
        "\n",
        "###############\n",
        "# Environment #\n",
        "###############\n",
        "env_params = GymEnvironmentParameters(level='short_corridor_env:ShortCorridorEnv')\n",
        "\n",
        "\n",
        "#################\n",
        "# Graph Manager #\n",
        "#################\n",
        "graph_manager = BasicRLGraphManager(agent_params=agent_params,\n",
        "                                    env_params=env_params,\n",
        "                                    schedule_params=schedule_params)\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting short_corridor_dqn_preset.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mCwoQwnGHZL5"
      },
      "source": [
        "##### ***2.4 Run new preset***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4db-w4ETHZL6",
        "colab": {}
      },
      "source": [
        "!coach -p /content/short_corridor_dqn_preset.py:graph_manager\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "aMs0JnUBHZL8"
      },
      "source": [
        "### ***3 Adding a new agent***\n",
        "Coach modularity makes adding an agent a clean and simple task.\n",
        "Typically consists of four parts:\n",
        "\n",
        "\n",
        "1.   Implement an agent's specific network head (and loss)\n",
        "2.   Implement exploration policy (optional)\n",
        "3.   Define new parameters class that extends `AgentParameters`\n",
        "4.   Implement a preset to run the agent on some environment\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MtfJT8XbmsUo",
        "colab_type": "text"
      },
      "source": [
        "##### ***3.1 Write stochastic output layer***\n",
        "We use stochastic policy, meaning that we only produce the probability of going left and going right.\n",
        "This layer takes in the input from previous layer, the middleware, and outputs two numbers. \n",
        "\n",
        "![Probabilistic output](https://drive.google.com/uc?id=1hB_AsKUlxlu43sMkPAFfLaK6Z5sz1I-n)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KIYd8t8GnCSu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile probabilistic_layer.py\n",
        "import tensorflow as tf\n",
        "from rl_coach.architectures.tensorflow_components.layers import Dense\n",
        "\n",
        "class ProbabilisticLayer(object):\n",
        "    def __init__(self, input_layer, num_actions):\n",
        "        super().__init__()\n",
        "        scores = Dense(num_actions)(input_layer, name='logit')\n",
        "        self.event_probs = tf.nn.softmax(scores, name=\"policy\")\n",
        "        # define the distributions for the policy and the old policy\n",
        "        self.policy_distribution = tf.contrib.distributions.Categorical(probs=self.event_probs)\n",
        "\n",
        "    def log_prob(self, action):\n",
        "        return self.policy_distribution.log_prob(action)\n",
        "\n",
        "    def layer_output(self):\n",
        "        return self.event_probs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wNelbGV7HZME"
      },
      "source": [
        "##### ***3.2 Implement network head i.e. implement the loss***\n",
        "The Head needs to inherit from the base class `Head`.\n",
        "\n",
        "In order to maximize the sum of rewards, we want to go in the following direction $-\\Sigma_i R_i \\nabla_Wlog(\\pi(a_i|x_i))$\n",
        "\n",
        "`Complete code`\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hUnA2ZJHHZMF",
        "colab": {}
      },
      "source": [
        "%%writefile simple_pg_head.py\n",
        "import tensorflow as tf\n",
        "from rl_coach.architectures.tensorflow_components.heads.head import Head\n",
        "from rl_coach.base_parameters import AgentParameters\n",
        "from rl_coach.spaces import SpacesDefinition\n",
        "from probabilistic_layer import ProbabilisticLayer\n",
        "\n",
        "\n",
        "class SimplePgHead(Head):\n",
        "    def __init__(self, agent_parameters: AgentParameters,\n",
        "                 spaces: SpacesDefinition, network_name: str,\n",
        "                 head_idx: int = 0, is_local: bool = True):\n",
        "        super().__init__(agent_parameters, spaces, network_name)\n",
        "\n",
        "        self.exploration_policy = agent_parameters.exploration\n",
        "\n",
        "    def _build_module(self, input_layer):\n",
        "        # Define inputs\n",
        "        actions = tf.placeholder(tf.int32, [None], name=\"actions\")\n",
        "        returns = tf.placeholder(tf.float32, [None], name=\"returns\")\n",
        "\n",
        "        # Two actions, left or right\n",
        "        policy_distribution = ProbabilisticLayer(input_layer, num_actions=2)\n",
        "\n",
        "        # calculate loss\n",
        "        log_prob = policy_distribution.log_prob(actions)\n",
        "        # We only want to encourage good actions, so we multiply the log probability with ...\n",
        "        modulated_log_prob = returns*log_prob\n",
        "        expected_modulated_log_prob = tf.reduce_mean(modulated_log_prob)\n",
        "\n",
        "        ### Coach bookeeping\n",
        "        # List of placeholders for additional inputs to the stochastic head \n",
        "        #(except from the middleware input)\n",
        "        self.input.append(actions)\n",
        "        # The output of the stochastic head, which is also the output of the network.\n",
        "        self.output.append(policy_distribution.layer_output())\n",
        "        # Placeholder for the target that we will use to train the network\n",
        "        self.target = returns\n",
        "        # The loss that we will use to train the network.\n",
        "        # We take the gradient of this loss and move in the opposite direction\n",
        "        self.loss = -expected_modulated_log_prob\n",
        "        tf.losses.add_loss(self.loss)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "3Eqej3v50a91"
      },
      "source": [
        "##### ***3.3 Define exploration policy*** \n",
        "Every iteration we want to sample from the network output distribution i.e. toss a bias coin to get the agent's actual move\n",
        "\n",
        "**`Complete code`**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zuYBPgnI0a-A",
        "colab": {}
      },
      "source": [
        "%%writefile simple_pg_exploration.py\n",
        "\n",
        "import numpy as np\n",
        "from rl_coach.exploration_policies.exploration_policy import ExplorationPolicy, ExplorationParameters\n",
        "from rl_coach.spaces import ActionSpace\n",
        "\n",
        "\n",
        "class DiscreteExplorationParameters(ExplorationParameters):\n",
        "    @property\n",
        "    def path(self):\n",
        "        return 'simple_pg_exploration:DiscreteExploration'\n",
        "\n",
        "\n",
        "class DiscreteExploration(ExplorationPolicy):\n",
        "    \"\"\"\n",
        "    Discrete exploration policy is intended for discrete action spaces. It expects the action values to\n",
        "    represent a probability distribution over the action\n",
        "    \"\"\"\n",
        "    def __init__(self, action_space: ActionSpace):\n",
        "        \"\"\"\n",
        "        :param action_space: the action space used by the environment\n",
        "        \"\"\"\n",
        "        super().__init__(action_space)\n",
        "\n",
        "    def get_action(self, probabilities):\n",
        "        # choose actions according to the probabilities\n",
        "        chosen_action = np.random.choice(self.action_space.actions, p=probabilities)\n",
        "        return chosen_action, probabilities\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "XG5y-G43HZL9"
      },
      "source": [
        "##### ***3.4 Define new agent parameters***\n",
        "Coach is modular!\n",
        "\n",
        "Each class in Coach has a complementary parameters class which defines its constructor. \n",
        "This is also true for the agent. The agent has a complementary `AgentParameters` class. This class enables selecting the paramenters of the agent sub modules.\n",
        "\n",
        "It consists of the following four parts:\n",
        "\n",
        "\n",
        "\n",
        "1.   Algorithm\n",
        "2.   Exploration\n",
        "3.   Memory\n",
        "4.   Networks\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bTAcPbnsHZL-",
        "colab": {}
      },
      "source": [
        "%%writefile simple_pg_params.py\n",
        "from rl_coach.architectures.embedder_parameters import InputEmbedderParameters\n",
        "from rl_coach.architectures.head_parameters import HeadParameters\n",
        "from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters\n",
        "from rl_coach.base_parameters import NetworkParameters, AlgorithmParameters, \\\n",
        "    AgentParameters\n",
        "\n",
        "from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters\n",
        "from rl_coach.exploration_policies.categorical import CategoricalParameters\n",
        "from rl_coach.memories.episodic.single_episode_buffer import SingleEpisodeBufferParameters\n",
        "from rl_coach.spaces import DiscreteActionSpace, BoxActionSpace\n",
        "from rl_coach.agents.policy_optimization_agent import PolicyGradientRescaler\n",
        "from simple_pg_exploration import DiscreteExplorationParameters\n",
        "\n",
        "class SimplePgAgentParameters(AgentParameters):\n",
        "    def __init__(self):\n",
        "        super().__init__(algorithm=SimplePGAlgorithmParameters(),\n",
        "                         exploration=DiscreteExplorationParameters(),\n",
        "                         memory=SingleEpisodeBufferParameters(),\n",
        "                         networks={\"main\": SimplePgTopology()})\n",
        "    @property\n",
        "    def path(self):\n",
        "        #return 'simple_pg_agent:SimplePgAgent'\n",
        "        return 'rl_coach.agents.policy_gradients_agent:PolicyGradientsAgent'\n",
        "\n",
        "        \n",
        "    \n",
        "# Since we are adding a new head we need to tell coach the heads path\n",
        "class SimplePgHeadParams(HeadParameters):\n",
        "    def __init__(self):\n",
        "        super().__init__(parameterized_class_name=\"AiWeekHead\")\n",
        "\n",
        "    @property\n",
        "    def path(self):\n",
        "        return 'simple_pg_head:SimplePgHead'\n",
        "\n",
        "\n",
        "class SimplePgTopology(NetworkParameters):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.input_embedders_parameters = {'observation': InputEmbedderParameters()}\n",
        "        self.middleware_parameters = FCMiddlewareParameters()\n",
        "        self.heads_parameters = [SimplePgHeadParams()]\n",
        "\n",
        "\n",
        "class SimplePGAlgorithmParameters(AlgorithmParameters):\n",
        "    \"\"\"\n",
        "    :param num_steps_between_gradient_updates: (int)\n",
        "        The number of steps between calculating gradients for the collected data. In the A3C paper, this parameter is\n",
        "        called t_max. Since this algorithm is on-policy, only the steps collected between each two gradient calculations\n",
        "        are used in the batch.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        # TOTAL_RETURN\n",
        "        # FUTURE_RETURN\n",
        "        # FUTURE_RETURN_NORMALIZED_BY_EPISODE \n",
        "        # FUTURE_RETURN_NORMALIZED_BY_TIMESTEP\n",
        "        # Q_VALUE\n",
        "        # A_VALUE\n",
        "        # TD_RESIDUAL\n",
        "        # DISCOUNTED_TD_RESIDUAL\n",
        "        # GAE\n",
        "        self.policy_gradient_rescaler = PolicyGradientRescaler.FUTURE_RETURN\n",
        "        self.num_steps_between_gradient_updates = 20000  # this is called t_max in all the papers\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_8RT_QFdHZMH"
      },
      "source": [
        "##### ***3.5 Write preset to run new agent on short corridor***\n",
        "complete code\n",
        "* **complete code**\n",
        "* **Hint: look at DQN preset**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "5f2d6f52-ae36-4854-b4d3-d727f3e84b94",
        "id": "D1PLQuNaHZMI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile short_corridor_new_agent_preset.py\n",
        "from rl_coach.base_parameters import VisualizationParameters\n",
        "from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps\n",
        "from rl_coach.environments.gym_environment import GymEnvironmentParameters\n",
        "from rl_coach.filters.filter import NoInputFilter, NoOutputFilter\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import SimpleSchedule\n",
        "from rl_coach.memories.memory import MemoryGranularity\n",
        "from rl_coach.schedules import LinearSchedule\n",
        "from simple_pg_params import SimplePgAgentParameters\n",
        "\n",
        "\n",
        "####################\n",
        "# Graph Scheduling #\n",
        "####################\n",
        "schedule_params = SimpleSchedule()\n",
        "\n",
        "\n",
        "#########\n",
        "# Agent #\n",
        "#########\n",
        "agent_params = SimplePgAgentParameters()\n",
        "agent_params.input_filter = NoInputFilter()\n",
        "agent_params.output_filter = NoOutputFilter()\n",
        "\n",
        "\n",
        "###############\n",
        "# Environment #\n",
        "###############\n",
        "env_params = GymEnvironmentParameters(level='short_corridor_env:ShortCorridorEnv')\n",
        "\n",
        "#################\n",
        "# Graph Manager #\n",
        "#################\n",
        "graph_manager = BasicRLGraphManager(agent_params=agent_params,\n",
        "                                    env_params=env_params,\n",
        "                                    schedule_params=schedule_params)\n",
        "\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting short_corridor_new_agent_preset.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Ufc_EAe3HZMK"
      },
      "source": [
        "##### ***3.6 Run preset of the new agent on the new environment***\n",
        "\n",
        "**`Complete code`**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ad2xpEu8HZML",
        "colab": {}
      },
      "source": [
        "!coach -p /content/short_corridor_new_agent_preset.py:graph_manager"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}