validate_baseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "deep-coder baseline",
      "version": "0.3.2",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yje9hqtcUQ_f",
        "colab_type": "text"
      },
      "source": [
        "### Initialization\n",
        "* Check whether the runtime is host or local.\n",
        "* Mount Google Drive when using the host runtime."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FwqGy_GyUQnw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try:\n",
        "  from google.colab import drive\n",
        "  drive.mount('/gdrive')\n",
        "  runtime = \"host\"\n",
        "except:\n",
        "  runtime = \"local\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_S457sT6QMUr",
        "colab_type": "text"
      },
      "source": [
        "### Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QN-4eF51DNqt",
        "colab": {}
      },
      "source": [
        "#@title Parameters\n",
        "#@markdown |Name            |Description|\n",
        "#@markdown |:---            |:---|\n",
        "#@markdown |`seed`|The random seed|\n",
        "seed = 3984 #@param {type: \"number\"}\n",
        "\n",
        "#@markdown ### `deep-coder` Repositories\n",
        "#@markdown |Name            |Description|\n",
        "#@markdown |:---            |:---|\n",
        "#@markdown |`repository_url`|The URL of `deep-coder` git repository (enabled only in the host runtime)|\n",
        "#@markdown |`branch_name`   |The branch name (enabled only in the host runtime)|\n",
        "repository_url = \"https://github.com/HiroakiMikami/deep-coder\" #@param {type: \"string\"}\n",
        "branch_name = \"master\" #@param {type: \"string\"}\n",
        "\n",
        "#@markdown ### Validation Settings\n",
        "#@markdown |Name                |Description|\n",
        "#@markdown |:---                |:---|\n",
        "#@markdown |`timeout_second`    ||\n",
        "#@markdown |`max_program_length`|The maximum length of the program|\n",
        "timeout_second = 1 #@param {type: \"number\"}\n",
        "max_program_length = 2 #@param {type: \"number\"}\n",
        "\n",
        "#@markdown ### Filepath\n",
        "#@markdown |Name                |Description|\n",
        "#@markdown |:---                |:---|\n",
        "#@markdown |`train_dataset_path`|The file path of the training dataset.|\n",
        "#@markdown |`valid_dataset_path`|The file path of the validation dataset.|\n",
        "#@markdown |`destination_path`  |The directory of the directory that will contain the training results.|\n",
        "train_dataset_path = \"./dataset/train.pickle\" #@param {type: \"string\"}\n",
        "valid_dataset_path = \"./dataset/valid.pickle\" #@param {type: \"string\"}\n",
        "destination_path = \"./out/baseline\" #@param {type: \"string\"}\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_BembldCdOO1",
        "colab_type": "text"
      },
      "source": [
        "### Setup\n",
        "* Fix the random seed\n",
        "* Download the codebase\n",
        "  1. Clone git repository and move to the specified branch\n",
        "  2. Initialize submodule\n",
        "  3. Install chainer and cupy\n",
        "* Copy the dataset from Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GwjlAkY1fR5j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import random\n",
        "\n",
        "SEED_MAX = 2**32 - 1\n",
        "\n",
        "root_rng = np.random.RandomState(seed)\n",
        "random.seed(root_rng.randint(SEED_MAX))\n",
        "np.random.seed(root_rng.randint(SEED_MAX))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FIZJmuz8QFn_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if runtime == \"host\":\n",
        "  %cd /content\n",
        "  !rm -rf deep-coder\n",
        "  ![ ! -e deep-coder ] && git clone $repository_url deep-coder\n",
        "  %cd deep-coder\n",
        "  !git checkout origin/$branch_name\n",
        "  !git submodule init\n",
        "  !git submodule update\n",
        "  !make -C DeepCoder_Utils/enumerative-search -j `nproc`\n",
        "  !curl https://colab.chainer.org/install | sh -",
        "  !pip install tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_JEszRdKFzkb",
        "colab_type": "text"
      },
      "source": [
        "### Validate Baseline Predictor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fJ0rp0_BGeSb",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "import os\n",
        "import chainer as ch\n",
        "from chainer import datasets\n",
        "from src.dataset import EncodedDataset, Dataset\n",
        "import src.inference as I\n",
        "from src.model import ModelShapeParameters\n",
        "from tqdm import tqdm_notebook as tqdm\n",
        "\n",
        "with open(valid_dataset_path, \"rb\") as f:\n",
        "    dataset: Dataset = pickle.load(f)\n",
        "\n",
        "with open(train_dataset_path, \"rb\") as f:\n",
        "    train: ch.datasets.TupleDataset = pickle.load(f).dataset\n",
        "pred = I.predict_with_prior_distribution(train)\n",
        "\n",
        "results = dict([])\n",
        "num_succ = 0\n",
        "for i, (entry,) in enumerate(tqdm(dataset.dataset)):\n",
        "    result = I.search(\n",
        "        os.path.join(os.getcwd(), \"DeepCoder_Utils\",\n",
        "                     \"enumerative-search\", \"search\"),\n",
        "        timeout_second,\n",
        "        dataset.metadata.value_range,\n",
        "        entry.examples,\n",
        "        max_program_length,\n",
        "        pred\n",
        "    )\n",
        "    results[i] = result\n",
        "    if result.is_solved:\n",
        "        num_succ += 1\n",
        "\n",
        "print(\"Solved: {} of {} examples\".format(num_succ, len(dataset.dataset)))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YWufhkoaw9Bq",
        "colab_type": "text"
      },
      "source": [
        "### Teardown\n",
        "* Save the baseline result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-WYlqxVkO5i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import chainer as ch\n",
        "\n",
        "if not os.path.exists(destination_path):\n",
        "    os.makedirs(destination_path)\n",
        "\n",
        "with open(os.path.join(destination_path, \"result.pickle\"), \"wb\") as f:\n",
        "    pickle.dump(results, f)\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}