notebooks/Create Cropped Faces and Metadata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "from utils.imutil import *\n",
    "from utils.list_all_files import *\n",
    "from utils.crop import *\n",
    "from utils.progress import *\n",
    "from utils.mosaic import *\n",
    "from utils.draw_shapes import *\n",
    "\n",
    "import ujson\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = '../data/openface+microsoft/json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_filenames = list(list_all_files(input_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59278 0:00:12 5020.44/s\n"
     ]
    }
   ],
   "source": [
    "filenames = []\n",
    "reps = []\n",
    "rects = []\n",
    "ages = []\n",
    "genders = []\n",
    "\n",
    "for fn in progress(json_filenames):\n",
    "    with open(fn) as f:\n",
    "        try:\n",
    "            data = ujson.load(f)\n",
    "        except:\n",
    "            print(f.read())\n",
    "            break\n",
    "        \n",
    "    for face in data['facesFromOpenFace']:\n",
    "        try:\n",
    "            rects.append([face[key] for key in 'YBXR'])\n",
    "            reps.append(face['rep'])\n",
    "            ages.append(face['age'])\n",
    "            genders.append(face['gender'])\n",
    "            filenames.append(data['imageName'])\n",
    "        except KeyError:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_categories = 20\n",
    "age_limits = []\n",
    "for pct in np.linspace(0, 100, n_categories+1):\n",
    "    if pct == 0:\n",
    "        continue\n",
    "    limit = np.percentile(ages, pct)\n",
    "    age_limits.append(limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[6.099999904632568,\n",
       " 11.899999618530273,\n",
       " 17.700000762939453,\n",
       " 22.100000381469727,\n",
       " 25.100000381469727,\n",
       " 27.299999237060547,\n",
       " 29.0,\n",
       " 30.600000381469727,\n",
       " 32.20000076293945,\n",
       " 33.79999923706055,\n",
       " 35.599998474121094,\n",
       " 37.5,\n",
       " 39.599998474121094,\n",
       " 41.79999923706055,\n",
       " 44.20000076293945,\n",
       " 47.099998474121094,\n",
       " 50.70000076293945,\n",
       " 56.0,\n",
       " 65.0999984741211,\n",
       " 95.30000305175781]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnXuUFdWV/z9blIfKQxT9IS02GqJRiQ9YGn/6m4WYjORhTGaNETNj0HEGdUWjTiaKRoceH1kkMToxmTBigqgT3yYGFY3G6DIaMTY+EMQHkBZbGGlEEJG3+/fHreZemqrqU3Wq+lZ1789arL73u79nn3PPfWzqcapEVTEMwzAMF3aq9wAMwzCM8mBFwzAMw3DGioZhGIbhjBUNwzAMwxkrGoZhGIYzVjQMwzAMZ6xoGIZhGM5Y0TAMwzCcsaJhGIZhOLNzvQeQlr322ksbGxvrPQyWtK0D4IAhuxUij08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qWcz/3LlzV6rqkLTtS1s0GhsbaW5urvcwOO2m5wC4+5xjC5HHJ5dLuzhPVCxM99HS6HnFknh8/HnlKEOfRpUs5l9E3vYZg+2eMgzDMJwp7ZZGUbhg3MhC5fHJ5dIuzhMVC9N9tDR6XrEkHh9/XjnK0KdRpQjzL2W9yu2YMWO0CLunDMMwyoSIzFXVMWnb25aGJwuWrQHg0H0HFiKPTy6XdnGeqFiY7qOl0fOKJfH4+PPKUYY+O2Pz5s20trayYcOGeg8ldzZv/QSAXXp1fmShb9++NDQ0sMsuu2Q6Bisanlz14GuA/4HBrPL45HJpF+eJioXpPloaPa9YEo+PP68cZeizM1pbW+nfvz+NjY2ISL2HkyuL2z4C4MAhu8f6VJX333+f1tZWRowYkekY7EC4YRilZsOGDey5557dvmAkQUTYc889c9n6sqJhGEbpsYKxI3nNiRUNwzAMwxk7pmEYRrfiuKl/5N3V6zPLN2xQP56dPC6zfB156qmnuO6663jooYdy6yNLrGh4csn4gwqVxyeXS7s4T1is9gvcOPnhHeKdaXvt3pubzhjtPI6k4/ONJfH4+PPKUYY+k/Lu6vW0TP1yZvnCPqP14v8M6FvvIVjR8GX0/oMLlccnl0u7OE9YzPcL3Dj54dC8UeNIOj7fWBKPjz+vHGXoswy0tLQwfvx4jj/+eObMmcPhhx/OWWedxZQpU1ixYgW//vWvAbjoootYv349/fr145ZbbuGgg7YvwuvWreOCCy7g1VdfZcuWLTQ1NXHKKadsi+/Wp/4/2XZMw5O5b69i7turCpPHJ5dLuzhPVCxMd9XadZ++8owl8fj488pRhj7LwqJFi7jwwguZN28er7/+OnfccQfPPPMM1113HT/4wQ84+OCDefrpp3nppZe46qqruPzyy3fIce211zJu3DheeOEFnnzySb73ve+xbt26bfF1G7ewbuOWrnxZO2BFw5MfPfoGP3r0jcLk8cnl0i7OExUL0121dt2nrzxjSTw+/rxylKHPsjBixAhGjRrFTjvtxKGHHsqJJ56IiDBq1ChaWlpYs2YNp556KocddhgXX3wxCxYs2CHHY489xtSpUzniiCMYO3YsGzZsYOnSpdvi//vhBv73w/ouYqz/to5hGEY3oE+fPtse77TTTtue77TTTmzZsoUrr7ySE044gd/+9re0tLQwduzYHXKoKvfff/8Ou62KhG1pGIZhdAFr1qxh2LBhAMycOTPUc9JJJ/Gzn/2M9msCvvTSS101PGc63dIQkRnAV4AVqnpYoN0NtJfCQcBqVT1CRBqBhUD79uscVT03aDMamAn0A2YDF6qqishg4G6gEWgBvqGqH2Tw2gzD6IEMG9Qv0zOehg3ql0meSy65hIkTJ3L99dczblz4KbxXXnklF110EZ/97GdRVRobGwt3Kq7L7qmZwM+B29oFVT2t/bGI/ARYU+NfrKpHhOSZBkwC5lApGuOBR4DJwBOqOlVEJgfPL032MgzDMCrkuaYiisbGRubPn7/tee2WRG3szTff3KZfffXVAIwdO3bbrqp+/fpx00035T9gDzotGqr6dLAFsQNSWaf+DSD2XRKRocAAVX0ueH4b8DUqReMUYGxgvRV4ihIVjX8/+ZBC5fHJ5dIuzhMVC9NdtSy8ecaSeHz8eeUoQ59GlX0Hln+dxv8D3lPVt2q0ESLyEvAhcIWq/gkYBrTWeFoDDWAfVV0OoKrLRWTvqM5EZBKVrRWGDx/uOfRsyOoS0VleajptLpd2cZ6oWJjuqmXhzTOWxOPjzytHGfo0qvTrXf9zl3wPhJ8O3FnzfDkwXFWPBP4VuENEBgBhV85KfPcnVZ2uqmNUdcyQIanvi54pz7y1kmfeWlmYPD65XNrFeaJiYbqr1q779JVnLInHx59XjjL0aVRZu2EzazdsrusYUpctEdkZ+Dtg2zUeVHUjsDF4PFdEFgOfprJl0VDTvAFYFjx+T0SGBlsZQ4EVacdUD372x8pG1vEj9ypEHp9cLu3iPFGxMN1Va9ddcqYdn08sicfHn1eOMvRpVFmxdiMA/ftme2OlJPhsaXweeF1Vt+12EpEhItIreHwAMBJYEux+WisinwuOg3wL+F3QbBYwMXg8sUY3DMMwCkanRUNE7gSeAw4SkVYROTsITWD7XVMAfwPME5FXgPuAc1W1/ZoD5wG/BBYBi6kcBAeYCnxBRN4CvhA8NwzDMAqIy9lTp0foZ4Zo9wP3R/ibgcNC9PeBEzsbh2EYhhM3jII1Szv3uTJwOFz8aqzlxhtvZNq0aRx11FHbLk6YJU1NTey+++58feK5medOSv0PxRuGYWTJmqXQtKZznytNnZ8x9otf/IJHHnkk8/txFxErGp784O9GFSqPTy6XdnGeqFiY7qpl4c0zlsTj488rRxn6LDrnnnsuS5Ys4atf/SoTJkxg8eLFO1zafObMmTzwwANs3bqV+fPn893vfpdNmzZx++2306dPH2bPns3gwYO5+eabmT59Ops2beJTn/oUt99+O7vuuuu2voYN6seSxYsZf8Z3aGtrY9ddd+Xmm2/m4IMP7roXrKql/Dd69Gg1is/+lz5U1/ZG9+e1117bXpgyINsOHPLtv//+2tbWppdddpnefvvtqqr6wQcf6MiRI/Wjjz7SW265RQ888ED98MMPdcWKFTpgwACdNm2aqqpedNFFesMNN6iq6sqVK7fl/P73v6833nhjZQhTpuiPf/xjVVUdN26cvvnmm6qqOmfOHD3hhBMix7XD3Kgq0Kwev722peHJH157D4DPH7JPIfL45HJpF+eJioXprlq77pIz7fh8Ykk8Pv68cpShzzLx2GOPMWvWLK677jqA7S5tfsIJJ9C/f3/69+/PwIEDOfnkkwEYNWoU8+bNA2D+/PlcccUVrF69mo8++oiTTjppu/zL2j7gz3/+M6eeeuo2bePGjV3x0rZhRcOTm/+0BPD/EmWVxyeXS7s4T1QsTHfV2nWXnGnH5xNL4vHx55WjDH2WCY24tPnzzz/f6aXTAc4880weeOABDj/8cGbOnMlTTz21XZ4Va9fTf8BAXn755XxfSAx2aXTDMIyM8L20+dq1axk6dCibN28OPQurf/8BNAzfn3vvvReoFKlXXnnFf+AJsC0NwzC6FwOHO53xlCifI76XNr/66qs55phj2H///Rk1ahRr167dwXP9tF8x9Yp/45prrmHz5s1MmDCBww8/3LkPX6xoGIbRvehkTUUetLS0bHscdmnzM888kzPPPDPUXxs777zzOO+883Zo39TUBMDito/Yb/9GHn300SyGnQrbPWUYhmE4I+373srGmDFjtLm5ud7DYNnq9QDs63l3r6zy+ORyaRfnCYs1Tn6YPwc3xanVw7xx7Tv2FzWOpOPzjSXx+PjzylGGPjtj4cKFfOYzn6n3MLqETVs+AaD3zm7/3w+bGxGZq6pj0o7Bdk95ktWXJ8svYdpcLu3iPFGxMN1Vy8KbZyyJx8efV44y9OmCqlK5Fmr3xrVYAOS1QWC7pzx58JVlPPjKss6NXZTHJ5dLuzhPVCxMd9XadZ++8owl8fj488pRhj47o2/fvrz//vu5/UgWidUfb2L1x5s69akq77//Pn37Zn+nP9vS8OR/5rwNwMmH71uIPD65XNrFeaJiYbqr1q675Ew7Pp9YEo+PP68cZeizMxoaGmhtbaWtra3eQ8mdtuB+GkP69+nEWSmmDQ0NnfqSYkXDMIxSs8suu/SICwUCNN30HAB3n3NE3cZgu6cMwzAMZ6xoGIZhGM5Y0TAMwzCcsXUanqxaVzmTYfBuvQuRxyeXS7s4T1iscfLDvHjlF3bQw7xx7Tv2FzWOpOPzjSXx+PjzylGGPo0qWcy/rdOoM1l9ebL8EqbN5dIuzhMVC9NdtSy8ecaSeHz8eeUoQ59GlSLMf6e7p0RkhoisEJH5NVqTiLwrIi8H/75UE7tMRBaJyBsiclKNPj7QFonI5Bp9hIg8LyJvicjdIlL/WUnAvc3vcG/zO4XJ45PLpV2cJyoWprtq7bpPX3nGknh8/HnlKEOfRpUizL/LlsZM4OfAbR30G1T1ulpBRA4BJgCHAvsCfxCRTwfh/wK+ALQCL4jILFV9DfhhkOsuEflv4GxgWsrX0+XcN7cVgFPH7FeIPD65XNrFeaJiYbqr1rvXTnzvvnl87755oeOJ04cN6sezwSVM0o69s1gSj48/rxxl6NOoUoT577RoqOrTItLomO8U4C5V3Qj8VUQWAUcHsUWqugRARO4CThGRhcA44JuB51agiRIVDSNfjhw+CIC7zzl2O/20beerR+uNkx/ughEaRs/C5+yp80VkXrD7ao9AGwbUbju1BlqUviewWlW3dNANwzCMApK2aEwDDgSOAJYDPwn0sCuGaQo9FBGZJCLNItLcEy4ZYBiGUTRSFQ1VfU9Vt6rqJ8DNVHdBtQK1O9sagGUx+kpgkIjs3EGP6ne6qo5R1TFDhgxJM3TDMAzDA6d1GsExjYdU9bDg+VBVXR48vhg4RlUniMihwB1Uisi+wBPASCpbFG8CJwLvAi8A31TVBSJyL3B/zYHwear6i87GVJR1Gus3bQWgX+9ehcjjk8ulXZwnLNY4+WEWXjV+Bz3M66q56o2TH6Zl6pdTj90llsTj488rRxn6NKpkMf+5r9MQkTuBscBeItIKTAHGisgRVHYltQDnAARF4B7gNWAL8G1V3RrkOR/4PdALmKGqC4IuLgXuEpFrgJeAX6V9MfUgqy9Pll/CtLlc2sV5omJhuo+WRs8rlsTj488rRxn6NKoUYf5dzp46PUSO/GFX1WuBa0P02cDsEH0J1d1bpeP251oAOOPYxkLk8cnl0i7OExUL0320NHpesSQeH39eOcrQp1GlCPNv157y5KF5y3lo3vLC5PHJ5dIuzhMVC9N9tDR6XrEkHh9/XjnK0KdRpQjzb0XDMAzDcMaKhmEYhuGMFQ3DMAzDGSsahmEYhjN2Pw0jVzqulegpfRtGUfFdp2FbGoZhGIYzVjQ8mf70YqY/vbgweXxyubSL80TFwnQfLY2eVyyJx8efV44y9GlUKcL8W9Hw5ImFK3hi4YrC5PHJ5dIuzhMVC9N9tDR6XrEkHh9/XjnK0KdRpQjzb0XDMAzDcMaKhmEYhuGMFQ3DMAzDGZd7hBsx9N0lm6tOZpXHJ5dLuzhPVCxM99HS6HnFknh8/HnlKEOfRpUizL+t0zByxdZpGEaxsHUahmEYRpdhRcOTG594ixufeKsweXxyubSL80TFwnQfLY2eVyyJx8efV44y9GlUKcL8W9Hw5NlFK3l20crC5PHJ5dIuzhMVC9N9tDR6XrEkHh9/XjnK0KdRpQjzb0XDMAzDcMbOnjJy5Zk+3+G7LedWnjSNrwY2XuGmbb0GGlIfszMMI2M6LRoiMgP4CrBCVQ8LtB8DJwObgMXAWaq6WkQagYXAG0HzOap6btBmNDAT6EflXuEXqqqKyGDgbqARaAG+oaofZPPyjHrTICuh8fjKk3PWVAM3PeemXfbjfAdoGEYiXLY0ZgI/B26r0R4HLlPVLSLyQ+Ay4NIgtlhVjwjJMw2YBMyhUjTGA48Ak4EnVHWqiEwOnl8a0r6Q7LFr70Ll8cnl0i7OExUL0501+QgcvWnH5xNL4vHx55WjDH0aVYow/07rNIItiIfatzQ6xL4O/L2q/kOUT0SGAk+q6sHB89OBsap6joi8ETxeHvieUtWDOhuTrdMoCU0DoWlN574c2h839Y+8u3p9qrbDBvXj2cnjUrU1jCLju04ji2Ma/0Rl91I7I0TkJeBD4ApV/RMwDGit8bQGGsA+qrocICgce2cwJsPw+tFvnPxwhiMxjO6D19lTIvJ9YAvw60BaDgxX1SOBfwXuEJEBgIQ0T7wUXUQmiUiziDS3tbWlHXam/PDR1/nho68XJo9PLpd2cZ6oWJjurG0+zTln2vH5xJJ4fPx55ShDn0aVIsx/6i0NEZlI5QD5iRrs41LVjcDG4PFcEVkMfJrKlkVDTfMGYFnw+D0RGVqzeyryYvGqOh2YDpXdU2nHniUvvp3NMfus8vjkcmkX54mKhenO2icjwdGbdnw+sSQeH39eOcrQp1GlCPOfaktDRMZTOVj9VVX9uEYfIiK9gscHACOBJcHup7Ui8jkREeBbwO+CZrOAicHjiTW6YRiGUTBcTrm9ExgL7CUircAUKmdL9QEer9SAbafW/g1wlYhsAbYC56rqqiDVeVRPuX0k+AcwFbhHRM4GlgKnZvLKDMMwjMzptGio6ukh8q8ivPcD90fEmoEdzr5S1feBEzsbh1E/ws5CijtQXBtr6evZ+c59oeWZ7Rf8QfhCwI76wOFw8aueAzAMoxZbEe7J0IG+v4rZ5vHJFdXu3dXrt11i/KK7XgLgPyccuYMvNNYUntdZOzRYGDjhe9vrQV+xetPATvP7xpJ4fPx55ShDn0aVIsy/3U/D6BSv+1L4rtPwwaNvuxeH0V2x+2kYhmEYXYYVDU/+48EF/MeDCwqTxyeXS7s4T1QsTPfR0uh5xZJ4fPx55ShDn0aVIsy/HdPw5LVlHxYqj08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qRZh/29IwDMMwnLGiYRiGYThjRcMwDMNwxo5peHLAkN0Klccnl0u7OE9ULEz30dLoecWSeHz8eeUoQ59GlSLMv63TMDrF1mkYRvfB1mkYhmEYXYYVDU8u+808LvvNvMLk8cnl0i7OExUL0320NHpesSQeH39eOcrQp1GlCPNvxzQ8WdK2rlB5fHK5tIvzRMXCdB8tjZ5XLInHx59XjjL0aVQpwvxb0TCMEIYN6rfDlXxdbgHbOPlhu7+40a2xomEYIdT+6J9203MA3H3OsZH+Wo/dX9zoztgxDcMwDMMZ29Lw5JB9BxQqj08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qRZh/W6dhdErrlANpkJXpGtfz7nl1WiNiazyMIuO7TsO2NIxOaZCV9VugZxhGoXA6piEiM0RkhYjMr9EGi8jjIvJW8HePQBcRuVFEFonIPBE5qqbNxMD/lohMrNFHi8irQZsbRUSyfJF5ctFdL227zWkR8vjkcmkX54mKhek+Who9r1gSj48/rxxl6NOoUoT5d93SmAn8HLitRpsMPKGqU0VkcvD8UuCLwMjg3zHANOAYERkMTAHGAArMFZFZqvpB4JkEzAFmA+OBR/xeWtewfM2GQuXxyeXSLs4TFQvTfbQ0el6xJB4ff145ytCnUaUI8++0paGqTwOrOsinALcGj28Fvlaj36YV5gCDRGQocBLwuKquCgrF48D4IDZAVZ/TygGW22pyGYZhGAXC55TbfVR1OUDwd+9AHwa8U+NrDbQ4vTVENwzDMApGHus0wo5HaAp9x8Qik0SkWUSa29raPIZoGIZhpMHn7Kn3RGSoqi4PdjGtCPRWYL8aXwOwLNDHdtCfCvSGEP8OqOp0YDpUTrn1GHtmHLX/HoXK45PLpV2cJyoWpvtoafS8Ykk8Pv68cpShT6NKEebfeZ2GiDQCD6nqYcHzHwPv1xwIH6yql4jIl4HzgS9RORB+o6oeHRwInwu0n031IjBaVVeJyAvABcDzVA6E/0xVZ8eNx9ZpdCH1vCeGD7ZOwzB2oEvWaYjInVS2EvYSkVYqZ0FNBe4RkbOBpcCpgX02lYKxCPgYOAsgKA5XAy8EvqtUtf3g+nlUztDqR+WsqVKcOWUYhtHTcCoaqnp6ROjEEK8C347IMwOYEaI3A4e5jKVonHv7XAD++4zRhcjjk8ulXZwnKham+2hp9LxiSTw+/rxylKFPo0oR5t9WhHvywcebCpXHJ5dLuzhPVCxM99Gc9YHDK7uo2mMbr6g8aNrxsuU7xGouf+I7L1n488pRhj6NKkWYfysaRvel4zWvgsuXc07IcY6OsZpiYxhGFbs0umEYhuGMFQ3DMAzDGds95clxn9qrUHl8crm0i/NExcJ0Hy2NnlcsicfHn1eOMvRpVCnC/Nv9NIzOKes6DR88XrOt0zCKjO86Dds9ZRiGYThjRcOTiTP+wsQZfylMHp9cLu3iPFGxMN1HS6PnFUvi8fHnlaMMfRpVijD/dkzDkw2btxYqj08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qRZh/29IwDMMwnLGiYRiGYThjRcMwDMNwxo5peHLiZ/bu3NSFeXxyubSL80TFwnQfLY2eVyyJx8efV44y9GlUKcL82zoNo3NsnUYibJ2GUWRsnYZhGIbRZVjR8OS0m57jtPYrpBYgj08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qRZh/KxqGYRiGM1Y0DMMwDGesaBiGYRjOWNEwDMMwnEm9TkNEDgLurpEOAP4dGAT8C9AW6Jer6uygzWXA2cBW4Duq+vtAHw/8FOgF/FJVp6YdV1fzlc8OLVQen1wu7eI8UbEw3UdLo+cVS+Lx8eeVowx9GlWKMP+ZrNMQkV7Au8AxwFnAR6p6XQfPIcCdwNHAvsAfgE8H4TeBLwCtwAvA6ar6Wlyftk6jC7F1Gok4buofeXf1+lRthw3qx7OTx6Vqaxgu+K7TyGpF+InAYlV9W0SiPKcAd6nqRuCvIrKISgEBWKSqSwBE5K7AG1s0isL6TZWrTvbr3asQeXxyubSL80TFwnQfLY2eVyzM4/KjH5WzcfLDnbZNMq6sqUefRpUizH9WxzQmUNmKaOd8EZknIjNEZI9AGwa8U+NpDbQofQdEZJKINItIc1tbW5ilyznzlr9w5i3+17fPKo9PLpd2cZ6oWJjuo6XR84ol8fj488pRhj6NKkWYf++iISK9ga8C9wbSNOBA4AhgOfCTdmtIc43RdxRVp6vqGFUdM2TIEK9xG4ZhGMnJYvfUF4EXVfU9gPa/ACJyM/BQ8LQV2K+mXQOwLHgcpRuGYRgFIouicTo1u6ZEZKiqLg+efh2YHzyeBdwhItdTORA+EvgLlS2NkSIygsrB9AnANzMYl2GkZ+DwysFwgI1XVP42jY/213oGDoeLX813fIZRJ7yKhojsSuWsp3Nq5B+JyBFUdjG1tMdUdYGI3EPlAPcW4NuqujXIcz7weyqn3M5Q1QU+4zIMb2p/9Nuv9XNOzNlUtZ72YmMY3RCvoqGqHwN7dtDOiPFfC1wbos8GZvuMpV78/eiGQuXxyeXSLs4TFQvTfbQ0el6xJB4ff145ytCnUaUI82/30zA6pyeu0/DB7sVhFBi7n0adWbVuE6vWbSpMHp9cLu3iPFGxMN1HS6PnFUvi8fHnlaMMfRpVijD/drtXT877n7kA3H3OsYXIE5nrhlGwZml8u+Bg7t19rtlOb9W9aN8ojhtnVCxM99HS6HnFknh8/HnlKEOfRpUizL8VjZ7CmqU0brjDydrRN2xQP57NY0yGYZQOKxoJibqukMvlH+p9XaHO9pW33xHM/hdpGEYUVjQS8u7q9dv9+Cb5oU1yXSHDMIwiYgfCDcMwDGdsS8OTf/zc/oXK45PLpV2cJyoWpvtoafS8Ykk8Pv68cpShT6NKEebf1mkkxOc8+rqeg29rLboOW6dhFBhbp1Fnlq1ez7KUN9zJI49PLpd2cZ6oWJjuo6XR84ol8fj488pRhj6NKkWYf9s95cnFd78M+J9xlFUen1wu7eI8UbEw3UdLo+cVS+Lx8eeVowx9GlWKMP89smj43o7TMAyjp9Iji0bH02YNwzAMN+yYhmEYhuGMFQ3DMAzDmR65eypL/uX/HVCoPD65XNrFeaJiYbqPlkbPK5bE4+PPK0cZ+jSqFGH+e+Q6jXqdC2/rNHoItk7DKDC2TqPOLG77iMVtHxUmj08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qRZh/2z3lyeW/qdxL2ve86azy+ORyaRfniYqF6T5aGj2vWBKPi3/YoH5eF7bM+0rKWX5OjeQUYf69i4aItABrga3AFlUdIyKDgbuBRqAF+IaqfiAiAvwU+BLwMXCmqr4Y5JkIXBGkvUZVb/Udm2GUjSQ/+GFXWLYrKRt5k9XuqRNU9Yia/WSTgSdUdSTwRPAc4IvAyODfJGAaQFBkpgDHAEcDU0Rkj4zGZhiGYWREXsc0TgHatxRuBb5Wo9+mFeYAg0RkKHAS8LiqrlLVD4DHgfE5jc0wDMNISRZFQ4HHRGSuiEwKtH1UdTlA8HfvQB8GvFPTtjXQovTtEJFJItIsIs1tbW0ZDN0wDMNIQhYHwo9T1WUisjfwuIi8HuOVEE1j9O0F1enAdKiccptmsFlzwbiRhcrjk8ulXZwnKham+2hp9LxiSTw+/rxylKFPo0oR5t+7aKjqsuDvChH5LZVjEu+JyFBVXR7sfloR2FuB/WqaNwDLAn1sB/0p37F1BceP3KvL8vhcaLGlbzZjiPNExcJ0Hy2NnlcsicfHn1eOMvRpVCnC/HsVDRHZDdhJVdcGj/8WuAqYBUwEpgZ/fxc0mQWcLyJ3UTnovSYoLL8HflBz8Ptvgct8xtZVLFhWWcR16L4Dc8/jeqHF0FxN2YwhzhMVC9N9tDR6XrFQz8DhlQV+MSz4pHIHtkN3env7wMDhcPGrsW2TjCtr6tGnUaUI8++7pbEP8NsDWBYVAAAMmUlEQVTKmbTsDNyhqo+KyAvAPSJyNrAUODXwz6Zyuu0iKqfcngWgqqtE5GrghcB3laqu8hxbl3DVg68BbudNZ3EOftZjStouzhMVC9N9tDR6XrFQj8OP/lUhp8sCnRabpOPKmnr0aVQpwvx7FQ1VXQIcHqK/D5wYoivw7YhcM4AZPuMpOnHn4Iedc28YhlE07DIihmEYhjN2GZEyccMoWLO0c9/GYGF9U81Sl4HD8xmTYRg9CisaZWLNUrerpwa7ujjHrmprGEa2WNHw5JLxBxUqj08ul3ZxnqhYmO6jpdHziiXx+PjzylGGPo0qRZh/u59GmbB7YnR/PN/j0n62jS7D7qdRZ+a+vYq5b/ufHZxVHp9cLu3iPFGxMN1HS6PnFUvi8fHnlaMMfRpVijD/tnvKkx89+gbgf6psVnl8crm0i/NExcJ0Hy2NnlcsicfH75rDZy2Qy704svycGskpwvxb0TCMboTPDZjsXhyGC7Z7yjAMw3DGioZhGIbhjBUNwzAMwxk7puHJv598SKHy+ORyaRfniYqF6T5aGj2vWBKPjz+vHGXo06hShPm3ouFJVpcozvJSx2lzubSL80TFwnQfLY2eVyyJx8efV44y9GlUKcL82+4pT555ayXPvLWyMHl8crm0i/NExcJ0Hy2NnlcsicfHn1eOMvRpVCnC/NuWhic/++NbgP8dtbLK45PLpV2cJyoWpvtoafS8Ykk8Pv68cpShT6NKEebftjQMwzAMZ2xLoyuJu7R52OXMO2KXNzcMo85Y0ehK4i5tbpczNwyjBFjRMIwiMXC4+33Co2625XCPcsNIjaqm+gfsBzwJLAQWABcGehPwLvBy8O9LNW0uAxYBbwAn1ejjA20RMNml/9GjR2ta9r/0odRtO7JoxVpdtGKtm3nKgGzyZDmmhO3iPFGxMN1HS6PnFUvi8fE754j5jHWGy/ciy8+pkZws5h9o1pS/+6rqtaWxBfiuqr4oIv2BuSLyeBC7QVWvqzWLyCHABOBQYF/gDyLy6SD8X8AXgFbgBRGZpaqveYytyzhwyO6FyuOTy6VdnCcqFqb7aGn0vGJJPD7+vHLUkvcVcg1/sn7P05C6aKjqcmB58HitiCwEhsU0OQW4S1U3An8VkUXA0UFskaouARCRuwJvKYrGH157D4DPH7JPIfL45HJpF+eJioXpPloaPa9YEo+PP68ctbj86Ef1aVfI7Rqyfs/TkMkxDRFpBI4EngeOA84XkW8BzVS2Rj6gUlDm1DRrpVpk3umgH5PFuHKhwxlQNwf7lT/f55rO28ac/XTzn5ZU8mTwYUiby6VdnCcqFqb7aGn0vGJJPD7+vHKUoU+jShHm37toiMjuwP3ARar6oYhMA64GNPj7E+CfAAlproSvFQm9B62ITAImAQwfXqfTTzueAWVnPRmG0YPwWtwnIrtQKRi/VtXfAKjqe6q6VVU/AW6muguqlcrB83YagGUx+g6o6nRVHaOqY4YMGeIzdMMwDCMFqYuGiAjwK2Chql5fow+tsX0dmB88ngVMEJE+IjICGAn8BXgBGCkiI0SkN5WD5bPSjsswDMPID5/dU8cBZwCvisjLgXY5cLqIHEFlF1MLcA6Aqi4QkXuoHODeAnxbVbcCiMj5wO+BXsAMVV3gMS7DMAwjJ6Ry2m75GDNmjDY3N6dq2zj5YVqmfjldx00DtzumsWz1egD2HdQvXb6M8/jkcmkX54mKhek+Who9r1gSj4/fOUeHz2fWRI3b6ztlOJPF50ZE5qrqmLTtbUW4J1n8yGeZxyeXS7s4T1QsTPfR0uh5xZJ4fPx55ShDn0aVIsy/FQ1PHnylcsz+5MP3LUQen1wu7eI8UbEw3UdLo+cVS+Lx8eeVI6s+bWFg11CP97wjPbJoPNPnO9D0zXSNO6y1+J85bwP+b2JWeXxyubSL80TFwnQfLY2eVyyJx8fvnCPJtas64nDdqqhx+/zo28JAd7L8nUhLjywaDbIy1/2+hlE3fC5WmLbYGD0KuwmTYRiG4YwVDcMwDMOZHrl7yjCM4uBzEL29vR1I7zp65DqNLM9lX7VuEwCDd+tdiDw+uVzaxXmiYmG6j5ZGzyuWxOPjzyvHdjh8LzLvMwN60hqRLObf1mnUmay+PFl+CdPmcmkX54mKhek+Who9r1gSj48/rxzb4XDm1eC4tnbHwNwpQrG2ouHJvc2Vq7qfOma/Tpxdk8cnl0u7OE9ULEz30dLoecWSeHz8eeXYDocf/cg+7cyrLiHz9zwFVjQ8uW9uK+D/JmaVxyeXS7s4T1QsTPfR0uh5xZJ4fPx55ShDn53RkxYWFmH+rWgYhlFqbGFh12JFwzAMf3JeiW4UBysahmH44/Ojf8Mov2MiVnS6FCsahmHUF98ffDsI36XYOg1P1m/aCkC/3r0Kkccnl0u7OE9ULEz30dLoecWSeHz8eeUoQ5+dcsMoWLM0VdPlDOHYDT9N1bYeB9GzmH9bp1FnsvryZPklTJvLpV2cJyoWpvtoafS8Ykk8Pv68cpShz07x2FIZ2jQw9cLAehxEL8L8W9Hw5PbnWgA449jGQuTxyeXSLs4TFQvTfbQ0el6xJB4ff145ytBnrngcwH+mz15A165EL8L8W9Hw5KF5ywH/NzGrPD65XNrFeaJiYbqPlkbPK5bE4+PPK0cZ+swVj62UXk2fSl1wljOEoU2LErcrwvwXpmiIyHjgp0Av4JeqOrXOQzIMw4gkzY9+tW15D94X4tLoItIL+C/gi8AhwOkickh9R2UYhmF0pBBFAzgaWKSqS1R1E3AXcEqdx2QYhmF0oCi7p4YB79Q8bwWOqdNYDMMwcqVyTCPFLqqNV7CJXYBjMx+TK4VYpyEipwInqeo/B8/PAI5W1Qs6+CYBk4KnBwFvpOxyL2BlyrbdAXv99vrt9fdM9gJ2U9UhaRMUZUujFai9bGMDsKyjSVWnA9N9OxORZp/FLWXHXr+9fnv9PfP1B6+90SdHUY5pvACMFJERItIbmADMqvOYDMMwjA4UYktDVbeIyPnA76mccjtDVRfUeViGYRhGBwpRNABUdTYwu4u6897FVXLs9fds7PX3XPx37xfhQLhhGIZRDopyTMMwDMMoAT2uaIjIeBF5Q0QWicjkeo8nT0RkPxF5UkQWisgCEbkw0AeLyOMi8lbwd496jzVPRKSXiLwkIg8Fz0eIyPPB6787OPmiWyIig0TkPhF5PfgcHNuT3n8RuTj47M8XkTtFpG93fv9FZIaIrBCR+TVa6PstFW4MfgvnichRLn30qKLRAy9XsgX4rqp+Bvgc8O3g9U4GnlDVkcATwfPuzIXAwprnPwRuCF7/B8DZdRlV1/BT4FFVPRg4nMo89Ij3X0SGAd8BxqjqYVROsplA937/ZwLjO2hR7/cXgZHBv0nANJcOelTRoIddrkRVl6vqi8HjtVR+MIZRec23BrZbga/VZ4T5IyINVK5f/cvguQDjgPsCS7d9/SIyAPgb4FcAqrpJVVfTg95/Kif79BORnYFdgeV04/dfVZ8GVnWQo97vU4DbtMIcYJCIDO2sj55WNMIuVzKsTmPpUkSkETgSeB7YR1WXQ6WwAHvXb2S585/AJcAnwfM9gdWquiV43p0/AwcAbcAtwe65X4rIbvSQ919V3wWuA5ZSKRZrgLn0nPe/naj3O9XvYU8rGhKidfvTx0Rkd+B+4CJV/bDe4+kqROQrwApVnVsrh1i762dgZ+AoYJqqHgmso5vuigoj2Hd/CjAC2BfYjcoumY501/e/M1J9F3pa0XC6XEl3QkR2oVIwfq2qvwnk99o3Q4O/K+o1vpw5DviqiLRQ2RU5jsqWx6BgdwV0789AK9Cqqs8Hz++jUkR6yvv/eeCvqtqmqpuB3wD/l57z/rcT9X6n+j3saUWjR12uJNh//ytgoapeXxOaBUwMHk8EftfVY+sKVPUyVW0IrrUzAfijqv4D8CTw94GtO7/+/wXeEZGDAulE4DV6yPtPZbfU50Rk1+C70P76e8T7X0PU+z0L+FZwFtXngDXtu7Hi6HGL+0TkS1T+t9l+uZJr6zyk3BCR44E/Aa9S3ad/OZXjGvcAw6l8sU5V1Y4Hz7oVIjIW+DdV/YqIHEBly2Mw8BLwj6q6sZ7jywsROYLKSQC9gSXAWVT+s9gj3n8R+Q/gNCpnEr4E/DOV/fbd8v0XkTuBsVSuZvseMAV4gJD3OyikP6dyttXHwFmq2txpHz2taBiGYRjp6Wm7pwzDMAwPrGgYhmEYzljRMAzDMJyxomEYhmE4Y0XDMAzDcMaKhmEYhuGMFQ3DMAzDGSsahmEYhjP/H8C0Ft1q6HErAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe98e8715f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([age for age,gender in zip(ages, genders) if gender == 'male'], bins=20, histtype='step', label='male')\n",
    "plt.hist([age for age,gender in zip(ages, genders) if gender == 'female'], bins=20, histtype='step', label='female')\n",
    "for age in age_limits:\n",
    "    plt.axvline(age, linestyle='--')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def natural_sort_key(s, _nsre=re.compile('([0-9]+)')):\n",
    "    return [int(text) if text.isdigit() else text.lower()\n",
    "            for text in _nsre.split(s)]\n",
    "\n",
    "faces_packed = list(zip(filenames, reps, rects, ages, genders))\n",
    "faces_packed.sort(key=lambda e: natural_sort_key(e[0]))\n",
    "filenames, reps, rects, ages, genders = zip(*faces_packed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_categories = []\n",
    "for age in ages:\n",
    "    for i, limit in enumerate(age_limits):\n",
    "        if age <= limit:\n",
    "            age_categories.append(i)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((31.899999618530273, 34.599998474121094, 36.099998474121094), [8, 10, 11])"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ages[:3], age_categories[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [],
   "source": [
    "onehots = np.zeros((len(filenames), 2 + len(age_limits)), dtype=np.float32)\n",
    "gender_age_categories = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (gender, age_category) in enumerate(zip(genders, age_categories)):\n",
    "    category = age_category\n",
    "    if gender == 'male':\n",
    "        onehots[i, 0] = 1\n",
    "    elif gender == 'female':\n",
    "        onehots[i, 1] = 1\n",
    "        category *= 2\n",
    "    onehots[i, age_category+2] = 1\n",
    "    gender_age_categories.append(category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/openface+microsoft/npy32/gender-age.npy', np.asarray(onehots).astype(np.float32))\n",
    "np.save('../data/openface+microsoft/npy32/gender-age-categories.npy', np.asarray(gender_age_categories).astype(np.int64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/openface+microsoft/npy32/openface-descriptors.npy', np.asarray(reps).astype(np.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "209355 0:12:19 283.27/s\n"
     ]
    }
   ],
   "source": [
    "output_side = 32\n",
    "# output_dtype = np.uint8\n",
    "\n",
    "def job(task):\n",
    "    try:\n",
    "        fn, rect = task\n",
    "        fn = '../data/photos/png1600/' + fn\n",
    "        \n",
    "        # rects were computed on 800x800 images\n",
    "        n,s,w,e = (np.asarray(rect) * 2).astype(int)\n",
    "        \n",
    "        # force it square\n",
    "        rows = s - n\n",
    "        cols = e - w\n",
    "        side = min(rows, cols)\n",
    "        s = n + side\n",
    "        e = w + side\n",
    "        \n",
    "        img = imread(fn)\n",
    "        \n",
    "        face_img = safe_crop(img, n, s, w, e, fill=0)\n",
    "        face_img = imresize(face_img, max_side=output_side)\n",
    "        \n",
    "        return face_img\n",
    "    \n",
    "    except:\n",
    "        raise\n",
    "\n",
    "results = progress_parallel(job, list(zip(filenames, rects)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/openface+microsoft/npy32/images.npy', np.asarray(results).astype(np.uint8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in np.random.randint(0, len(filenames), size=(4,)):\n",
    "#     imshow(results[i], zoom=4)\n",
    "#     print(genders[i])\n",
    "#     print(ages[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf3k]",
   "language": "python",
   "name": "conda-env-tf3k-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}