coco18_search_test_forVIT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "import json\n",
    "import copy\n",
    "import shutil\n",
    "import PIL\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, models, transforms, utils\n",
    "from torchvision.transforms.functional import to_tensor, normalize\n",
    "\n",
    "from tqdm import tqdm\n",
    "from core.dataset import COCODataset, COCODatasetWithID\n",
    "from core.model import Model\n",
    "from core.metrics import AccuracyLogger"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "read in training, validation, test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_data_path_test = './debug/coco_search18_fixations_TP_test.json'\n",
    "df_test = pd.read_json(paper_data_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['new_name'] = df_test['name'] + df_test['task']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "612"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_test.new_name.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "586"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_test.name.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./COCOstuff/annotations_UnRel_compatible/train.json', \"r\") as f:\n",
    "    project_json = json.load(f)\n",
    "    \n",
    "project_json.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_json['categories']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./debug/annotations.json\", \"r\") as f:\n",
    "    project_json = json.load(f)\n",
    "    \n",
    "project_json.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------\n",
      "Annotation Counts\n",
      "-------------------------------\n",
      "chair                        50\n",
      "fork                         46\n",
      "sink                         55\n",
      "tv                           56\n",
      "bowl                         28\n",
      "car                          20\n",
      "clock                        23\n",
      "cup                          55\n",
      "keyboard                     36\n",
      "knife                        28\n",
      "laptop                       24\n",
      "mouse                        21\n",
      "oven                         20\n",
      "potted plant                 30\n",
      "toilet                       31\n",
      "bottle                       33\n",
      "stop sign                    25\n",
      "microwave                    31\n",
      "Total                       612\n",
      "-------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dataset = COCODatasetWithID(\"./coco18/coco18_test.json\", './coco18/coco18_test', image_size =(224,224))\n",
    "dataloader = DataLoader(dataset, batch_size=32, num_workers=4, shuffle=True, pin_memory=True, drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "image, target_image, bbox_relative, label, annotation_id = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./coco18/coco18_test/000000340934.jpg'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.id2file[dataset.annotations[0]['image_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = Image.open('./coco18/coco18_test/000000340934.jpg')\n",
    "image = image.convert(\"RGB\")\n",
    "image = to_tensor(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 1050, 1680])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chair'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.idx2label[label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'bottle',\n",
       " 1: 'bowl',\n",
       " 2: 'car',\n",
       " 3: 'chair',\n",
       " 4: 'clock',\n",
       " 5: 'cup',\n",
       " 6: 'fork',\n",
       " 7: 'keyboard',\n",
       " 8: 'knife',\n",
       " 9: 'laptop',\n",
       " 10: 'microwave',\n",
       " 11: 'mouse',\n",
       " 12: 'oven',\n",
       " 13: 'potted plant',\n",
       " 14: 'sink',\n",
       " 15: 'stop sign',\n",
       " 16: 'toilet',\n",
       " 17: 'tv'}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.idx2label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "extract image feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def jsonfileprocessing(df):\n",
    "    df = df[['name', 'subject', 'task', 'bbox', 'correct']]\n",
    "    df = df[df['subject'] == 1]\n",
    "    df.drop(columns = ['subject', 'correct'], inplace = True)\n",
    "    df.reset_index(inplace = True, drop = True)\n",
    "\n",
    "    # to find out duplicate file names\n",
    "    counts = df['name'].value_counts()\n",
    "    name_duplicate = [k for k, v in counts.items() if v > 1]\n",
    "    df_duplicate = df[df[\"name\"].isin(name_duplicate)]\n",
    "    df_duplicate.reset_index(inplace = True, drop = True)\n",
    "\n",
    "    df_unique = df[~df[\"name\"].isin(name_duplicate)]\n",
    "    df_unique.reset_index(inplace = True, drop = True)\n",
    "    # return  \n",
    "    return df_unique, df_duplicate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "transfer COCO18 json file to COCO_STUFF style json file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def coco_dict(template_dict):\n",
    "    coco18_json = copy.deepcopy(template_dict)\n",
    "    for key in coco18_json.keys():\n",
    "        coco18_json[key].clear()\n",
    "        \n",
    "    return coco18_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def coco18_json_construct(coco18_json, df, df_duplicate, info, categories, startingIdx=0):\n",
    "      coco18_json['info'] = info\n",
    "      coco18_json['categories'] = categories\n",
    "      length = len(df)\n",
    "      lookup_table = {dict['name']: dict['id'] for dict in categories}\n",
    "      # construct image features\n",
    "      for index, row in df.iterrows():\n",
    "            annotation_temp = {\n",
    "                'area': 99999,\n",
    "                'iscrowd': 0,\n",
    "                'bbox': row['bbox'],\n",
    "                'image_id': startingIdx + index,\n",
    "                'category_id': lookup_table[row['task']]\n",
    "            }\n",
    "            image_temp = {\n",
    "                'height': 1050,\n",
    "                'width': 1680,\n",
    "                'file_name': row['name'],\n",
    "                'id': startingIdx + index\n",
    "            }\n",
    "            coco18_json['annotations'].append(annotation_temp)\n",
    "            coco18_json['images'].append(image_temp) \n",
    "\n",
    "      for index, row in df_duplicate.iterrows():\n",
    "      # initialization\n",
    "            annotation_temp = {\n",
    "                'area': 99999,\n",
    "                'iscrowd': 0,\n",
    "                'bbox': None,\n",
    "                'category_id': None,\n",
    "                'image_id': None\n",
    "            }\n",
    "            image_temp = {'file_name': None, 'height': 1050, 'width': 1680, 'id': None}\n",
    "\n",
    "            annotation_temp['bbox'] = row['bbox']\n",
    "            annotation_temp['image_id'] = startingIdx + length + index\n",
    "            annotation_temp['category_id'] = lookup_table[row['task']] \n",
    "\n",
    "            image_temp['file_name'] = row['task'] + row['name']\n",
    "            image_temp['id'] = startingIdx + length + index\n",
    "\n",
    "            coco18_json['annotations'].append(annotation_temp)\n",
    "            coco18_json['images'].append(image_temp)   \n",
    "\n",
    "\n",
    "      return coco18_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def copyfiles(src, dest, df, df_duplicate):\n",
    "    # sourcery skip: use-fstring-for-concatenation\n",
    "    # unique filename\n",
    "    for index, row in df.iterrows():\n",
    "        srcfile = src + row['task'] + \"/\" + row[\"name\"]\n",
    "        destfile = dest + \"/\" + row[\"name\"]\n",
    "        if os.path.exists(destfile):\n",
    "            continue\n",
    "        else:\n",
    "            shutil.copy(srcfile, destfile)\n",
    "        \n",
    "    for index, row in df_duplicate.iterrows():\n",
    "        srcfile = src + row['task'] + \"/\" + row[\"name\"]\n",
    "        destfile = dest + \"/\" + row['task'] + row[\"name\"] \n",
    "        if os.path.exists(destfile):\n",
    "            continue\n",
    "        else:\n",
    "            shutil.copy(srcfile, destfile)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict2jsonsave(json_dict, type_data):\n",
    "    json_str = json.dumps(json_dict, indent = 4)\n",
    "    filename = f'coco18_{type_data}.json'\n",
    "    with open(filename, 'w') as json_file:\n",
    "        json_file.write(json_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_train_duplicate = jsonfileprocessing(df_train)\n",
    "df_valid, df_valid_duplicate = jsonfileprocessing(df_valid)\n",
    "df_test, df_test_duplicate = jsonfileprocessing(df_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "processing training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coco18_train_json = coco_dict(project_json)\n",
    "info = {'description': 'COCO18 search dataset'}\n",
    "categories = [\n",
    "    {'id': 44, 'name': 'bottle'},\n",
    "    {'id': 51, 'name': 'bowl'},\n",
    "    {'id': 3, 'name': 'car'}, \n",
    "    {'id': 62, 'name': 'chair'}, \n",
    "    {'id': 85, 'name': 'clock'}, \n",
    "    {'id': 47, 'name': 'cup'}, \n",
    "    {'id': 48, 'name': 'fork'},  \n",
    "    {'id': 76, 'name': 'keyboard'}, \n",
    "    {'id': 49, 'name': 'knife'}, \n",
    "    {'id': 73, 'name': 'laptop'}, \n",
    "    {'id': 78, 'name': 'microwave'}, \n",
    "    {'id': 74, 'name': 'mouse'}, \n",
    "    {'id': 79, 'name': 'oven'}, \n",
    "    {'id': 64, 'name': 'potted plant'}, \n",
    "    {'id': 81, 'name': 'sink'}, \n",
    "    {'id': 13, 'name': 'stop sign'}, \n",
    "    {'id': 70, 'name': 'toilet'}, \n",
    "    {'id': 72, 'name': 'tv'}\n",
    "]\n",
    "coco18_train_json = coco18_json_construct(coco18_train_json, df_train, df_train_duplicate ,info, categories)\n",
    "source_folder = './coco18_images/'\n",
    "destination_folder = './coco18_train'\n",
    "copyfiles(source_folder, destination_folder, df_train, df_train_duplicate)\n",
    "dict2jsonsave(coco18_train_json, 'train')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coco18_valid_json = coco_dict(project_json)\n",
    "coco18_valid_json = coco18_json_construct(coco18_valid_json, df_valid, df_valid_duplicate, info, categories, 10000)\n",
    "source_folder = './coco18_images/'\n",
    "destination_folder = './coco18_valid'\n",
    "copyfiles(source_folder, destination_folder, df_valid, df_valid_duplicate)\n",
    "dict2jsonsave(coco18_valid_json, 'valid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coco18_test_json = coco_dict(project_json)\n",
    "coco18_test_json = coco18_json_construct(coco18_test_json, df_test, df_test_duplicate, info, categories, 20000)\n",
    "source_folder = './coco18_images/'\n",
    "destination_folder = './coco18_test'\n",
    "copyfiles(source_folder, destination_folder, df_test, df_test_duplicate)\n",
    "dict2jsonsave(coco18_test_json, 'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ff3c5d4e84e70bd69d08de6f672a9b55ed7e01a72700fa1f24565cb6d24c2072"
  },
  "kernelspec": {
   "display_name": "Python 3.9 (tensorflow)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}