notebooks/Create Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code ingests:\n",
    "\n",
    "- A `.npy` file of images or features to embed.\n",
    "- Optionally, a `.npy` file of corresponding labels/classifications for the images or features.\n",
    "\n",
    "And outputs:\n",
    "\n",
    "- A set of point clouds in `.npy` files, along with `colors.npy` based on the standardization approach [here](https://github.com/kylemcdonald/Coloring-t-SNE), and plots of the point clouds in a `plots/` folder.\n",
    "\n",
    "One way to make this better is to run UMAP with the standard parameters initially, and then to use that embedding as an initialization step to the remaining runs. This will encourage the other embeddings to be close in terms of interpolation distance, and might even speed up the optimization process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import umap\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from utils.progress import *\n",
    "from utils.rainbow import *\n",
    "from time import time\n",
    "import os\n",
    "\n",
    "def format_time(seconds):\n",
    "    minutes = int(seconds / 60)\n",
    "    seconds = seconds - 60 * minutes\n",
    "    if minutes > 0:\n",
    "        return f'{minutes}min {int(seconds)}s'\n",
    "    elif seconds > 0.01:\n",
    "        return f'{seconds:2.2f}s'\n",
    "    else:\n",
    "        return f'{seconds}s'\n",
    "    \n",
    "def plot_tsne(xy, colors=None, alpha=0.25, figsize=(6,6), s=0.5, cmap='hsv', filename=None):\n",
    "    plt.figure(figsize=figsize, facecolor='white')\n",
    "    plt.margins(0)\n",
    "    plt.axis('off')\n",
    "    fig = plt.scatter(xy[:,0], xy[:,1],\n",
    "                c=colors, # set colors of markers\n",
    "                cmap=cmap, # set color map of markers\n",
    "                alpha=alpha, # set alpha of markers\n",
    "                marker=',', # use smallest available marker (square)\n",
    "                s=s, # set marker size. single pixel is 0.5 on retina, 1.0 otherwise\n",
    "                lw=0, # don't use edges\n",
    "                edgecolor='') # don't use edges\n",
    "    # remove all axes and whitespace / borders\n",
    "    fig.axes.get_xaxis().set_visible(False)\n",
    "    fig.axes.get_yaxis().set_visible(False)\n",
    "    if filename is None:\n",
    "        plt.show()\n",
    "    else:\n",
    "        os.makedirs(os.path.split(filename)[0], exist_ok=True) \n",
    "        plt.savefig(filename)\n",
    "    \n",
    "def standardize(data):\n",
    "    out = np.copy(data).astype(np.float32)\n",
    "    out -= out.mean(axis=0)\n",
    "    std = out.std(axis=0)\n",
    "    std[np.where(std == 0)] = 1\n",
    "    out /= std\n",
    "    return out\n",
    "\n",
    "def job(task):\n",
    "    data, argmax, output_dir, n_neighbors, min_dist, y, target_metric = task\n",
    "    config = f'{min_dist:.3f}_{n_neighbors:02d}'\n",
    "    start = time()\n",
    "    if y is None:\n",
    "        embedder = umap.UMAP(min_dist=min_dist, n_neighbors=n_neighbors)\n",
    "        embedding = embedder.fit_transform(data)\n",
    "    else:\n",
    "        embedder = umap.UMAP(min_dist=min_dist, n_neighbors=n_neighbors, target_metric=target_metric)\n",
    "        embedding = embedder.fit_transform(data, y)\n",
    "    duration = time() - start\n",
    "    npy_path = os.path.join(output_dir, config + '.npy')\n",
    "    np.save(npy_path, embedding)\n",
    "    png_dir = os.path.join(output_dir, 'plots')\n",
    "    os.makedirs(png_dir, exist_ok=True)\n",
    "    png_path = os.path.join(png_dir, config + '.png')\n",
    "    plot_tsne(embedding, argmax, filename=png_path)\n",
    "    return f'{config}: {format_time(duration)}'\n",
    "\n",
    "def create_embedding(input_fn, output_dir,\n",
    "                     n_neighbors_opt=[2,3,5],\n",
    "                     min_dist_opt=[0.001, 0.01, 0.1],\n",
    "                     supervision_fn=None, target_metric='categorical'):\n",
    "    data = np.load(input_fn)\n",
    "    data = data.reshape(len(data), -1)\n",
    "    \n",
    "    y = None\n",
    "    if supervision_fn is not None:\n",
    "        y = np.load(supervision_fn)\n",
    "    \n",
    "    argmax = np.argmax(standardize(data), axis=1)\n",
    "    colors = to_rainbow(argmax)\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    np.save(os.path.join(output_dir, 'colors.npy'), colors)\n",
    "    \n",
    "    tasks = []\n",
    "    for n_neighbors in n_neighbors_opt:\n",
    "        for min_dist in min_dist_opt:\n",
    "            tasks.append((data, argmax, output_dir, n_neighbors, min_dist, y, target_metric))\n",
    "\n",
    "    timing = progress_parallel(job, tasks)\n",
    "    print(input_fn, output_dir)\n",
    "    print(timing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False False float32 (209355, 128) -0.406195 0.450748\n",
      "False False int64 (209355,) 0 38\n"
     ]
    }
   ],
   "source": [
    "for fn in ['../data/openface+microsoft/npy32/openface-descriptors.npy',\n",
    "          '../data/openface+microsoft/npy32/gender-age-categories.npy']:\n",
    "    x = np.load(fn)\n",
    "    print(np.isnan(x).any(), np.isinf(x).any(), x.dtype, x.shape, x.min(), x.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 0:02:32 0.06/s\n",
      "../data/openface+microsoft/npy32/openface-descriptors.npy ../data/embeddings/openface+microsoft\n",
      "['0.001_02: 16min 11s', '0.010_02: 15min 38s', '0.100_02: 16min 39s', '0.001_03: 14min 18s', '0.010_03: 14min 7s', '0.100_03: 14min 17s', '0.001_05: 14min 38s', '0.010_05: 14min 37s', '0.100_05: 14min 56s']\n"
     ]
    }
   ],
   "source": [
    "# openface+microsoft\n",
    "create_embedding(\n",
    "    '../data/openface+microsoft/npy32/openface-descriptors.npy',\n",
    "    '../data/embeddings/openface+microsoft',\n",
    "    supervision_fn='../data/openface+microsoft/npy32/gender-age-categories.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# depth images\n",
    "create_embedding(\n",
    "    '../data/depth/npy32/images.npy',\n",
    "    '../data/embeddings/depth')\n",
    "\n",
    "# saliency images\n",
    "create_embedding(\n",
    "    '../data/saliency/npy32/images.npy',\n",
    "    '../data/embeddings/saliency')\n",
    "\n",
    "# openface\n",
    "create_embedding(\n",
    "    '../data/openface/npy32/descriptors.npy',\n",
    "    '../data/embeddings/openface')\n",
    "\n",
    "# ellipses\n",
    "create_embedding(\n",
    "    '../data/face-ellipses/images.npy',\n",
    "    '../data/embeddings/face-ellipses')\n",
    "\n",
    "# detectron\n",
    "create_embedding(\n",
    "    '../data/detectron/npy32/images.npy',\n",
    "    '../data/embeddings/detectron')\n",
    "create_embedding(\n",
    "    '../data/detectron/npy32/images.npy',\n",
    "    '../data/embeddings/detectron-supervised',\n",
    "    supervision_fn='data/analysis/face_counts/categories.npy')\n",
    "\n",
    "# vgg features\n",
    "create_embedding(\n",
    "    '../data/dcnn/vgg/features_canonical.npy',\n",
    "    '../data/embeddings/vgg-features')\n",
    "create_embedding(\n",
    "    '../data/dcnn/vgg/features_canonical.npy',\n",
    "    '../data/embeddings/vgg-features-supervised',\n",
    "    supervision_fn='../data/analysis/face_counts/categories.npy')\n",
    "\n",
    "# inception features\n",
    "create_embedding(\n",
    "    '../data/dcnn/inceptionv3/features_canonical.npy',\n",
    "    '../data/embeddings/inceptionv3-features')\n",
    "create_embedding(\n",
    "    '../data/dcnn/inceptionv3/features_canonical.npy',\n",
    "    '../data/embeddings/inceptionv3-features-supervised',\n",
    "    supervision_fn='../data/analysis/face_counts/categories.npy')\n",
    "\n",
    "# inception predictions\n",
    "create_embedding(\n",
    "    '../data/dcnn/inceptionv3/predictions_canonical.npy',\n",
    "    '../data/embeddings/inceptionv3-predictions')\n",
    "create_embedding(\n",
    "    '../data/dcnn/inceptionv3/predictions_canonical.npy',\n",
    "    '../data/embeddings/inceptionv3-predictions-supervised',\n",
    "    supervision_fn='../data/analysis/face_counts/categories.npy')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}