2.4.2 分类任务数据集定义与配置.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 4217, 'valid': 100}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'数据预处理：增强'\n",
    "'在做迁移学习的时候，应该保持与(你正在用的)经典网络一致的数据预处理流程，以取得相近的训练结果'\n",
    "\n",
    "# import\n",
    "import torch.utils.data.dataloader as dataloader\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import os\n",
    "\n",
    "\n",
    "root = 'd:/GitRepos/data/Flower'\n",
    "datasets_names = ['train', 'valid']\n",
    "\n",
    "# data transform : preprocessing\n",
    "data_tranforms = {\n",
    "    'train' : transforms.Compose([\n",
    "     transforms.RandomRotation(45)          # 随机旋转，-45~45\n",
    "    ,transforms.CenterCrop(224)             # 从中心截取固定尺寸224的图片\n",
    "    ,transforms.RandomHorizontalFlip(0.5)   # 水平翻转，一半一半\n",
    "    ,transforms.RandomVerticalFlip(0.5)     # 垂直翻转\n",
    "                                            # 改变亮度、对比度、饱和度、色调 # 虽然pylance报语法错误但是可以正常运行\n",
    "    ,transforms.ColorJitter(brightness=0.2,contrast=0.1,saturation=0.1,hue=0.1)\n",
    "    ,transforms.RandomGrayscale(0.5)        # 随机转为灰度图,一半一半\n",
    "    ,transforms.ToTensor()                  # 转为Tensor\n",
    "                                            # 归一化\n",
    "    ,transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225))\n",
    "    ]),\n",
    "\n",
    "    'valid' : transforms.Compose([\n",
    "     transforms.Resize(256)                 # 输入图\n",
    "    ,transforms.CenterCrop(224)\n",
    "    ,transforms.ToTensor()\n",
    "    ,transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225))\n",
    "    ])\n",
    "}\n",
    "\n",
    "'从训练集中分出一部分作为测试集'\n",
    "# name e.g. daisy\n",
    "train_dirs = os.listdir(os.path.join(root,datasets_names[0]))\n",
    "# path e.g. d:/gitrepos/codevcprojects/data/flower/train/daisy\n",
    "subdirs = {datasets_names[0] : ['/'.join([root,datasets_names[0],train_dir]) for train_dir in train_dirs],\n",
    "           datasets_names[1] : ['/'.join([root,datasets_names[1],train_dir]) for train_dir in train_dirs]\n",
    "}\n",
    "# 创建valid子文件夹\n",
    "[os.makedirs(path) for path in subdirs[datasets_names[1]] if not os.path.exists(path)]\n",
    "# 移动文件\n",
    "[[os.rename(os.path.join(trainpath,img),os.path.join(validpath,img)) \n",
    "                for cnt,img in enumerate(os.listdir(trainpath),0)\n",
    "                if cnt < 20 and len(os.listdir(validpath))==0] \n",
    " for validpath,trainpath in zip(subdirs['valid'],subdirs['train'])]\n",
    "\n",
    "'制作数据集'\n",
    "# batch越大，则需要的显存也越大，报memory error的话，尝试降低batch_size\n",
    "batch_size = 8 # 16、32...需要8G以上\n",
    "\n",
    "image_datasets = {name : datasets.ImageFolder(root=os.path.join(root,name),transform=data_tranforms[name])\n",
    "        for name in datasets_names\n",
    "}\n",
    "image_loader = {\n",
    "    key : dataloader.DataLoader(\n",
    "    image_datasets[key],\n",
    "    batch_size= batch_size,\n",
    "    shuffle= True)\n",
    "    for key in image_datasets.keys()\n",
    "    }\n",
    "datasets_size = {key : len(image_datasets[key]) for key in image_datasets.keys()}\n",
    "class_names = image_datasets['train'].classes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d:/GitRepos/data/Flower\\train\n",
      "d:/GitRepos/data/Flower\\valid\n",
      "train\n",
      "valid\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "[print(entry.path) for entry in os.scandir('d:/GitRepos/data/Flower') if entry.is_dir()] # better performance than os.listdir()\n",
    "[print(entry) for entry in os.listdir('d:/GitRepos/data/Flower')]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0c12c839adeec326d421be117b103b29d3b0068db2c28a64ae1df63f05821871"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}