Book_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "books = pd.read_csv('files/book.csv')\n",
    "books"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "category = books['categoryName'].unique()\n",
    "count = books['categoryName'].nunique()\n",
    "print(category, count)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "category_entity = {}\n",
    "for i, v in enumerate(category):\n",
    "    category_entity[i] = v\n",
    "\n",
    "print(category_entity)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "raw",
   "source": [
    "----"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "authors = books['authors']"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i in range(len(books['authors'])):\n",
    "    if str(books['authors'][i])[-1]=='외':\n",
    "        authors.iloc[i] = books.loc[i,'authors'].split('외')[0]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "authors = authors[authors.notna()]\n",
    "authors"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "authors = authors[authors.str.contains(' 외 ')==False]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "authors = authors[authors.str.contains('엮은이|옮긴이')==False]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "author = list(authors['authors'])\n",
    "for i in range(len(authors)):\n",
    "    a = str(authors.loc[i, 'authors'])\n",
    "    if ',' in a :\n",
    "        author.remove(a)\n",
    "        a = str(a).split(',')\n",
    "        for k in a:\n",
    "            author.append(str(k))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "author_set = list(set(author))\n",
    "print(len(author_set))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for s in author_set:\n",
    "    s.strip()\n",
    "\n",
    "print(author_set)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print(len(set(author_set)))\n",
    "author = list(set(author_set))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "len(author)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "author_set = author.copy()\n",
    "author_dict = {}\n",
    "for i, v in enumerate(author_set):\n",
    "    author_dict[i] = v"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "author_dict"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "modified_dict = {key + 1000: value for key, value in author_dict.items()}\n",
    "print(len(modified_dict))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(modified_dict, orient='index').to_csv('author_entity.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "raw",
   "source": [
    "======"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "books_x = pd.read_csv('files/new_data/books.csv')\n",
    "boook = pd.merge(books, books_x[['id','publishedDate']], on='id', how='left')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book = boook.copy()\n",
    "book['publishedCent'] = [None] * len(book)\n",
    "for i in range(len(book)):\n",
    "    if str(book.loc[i, 'publishedDate_y'])[0:2] == '19':\n",
    "        book.loc[i, 'publishedCent'] = '20C'\n",
    "    elif str(book.loc[i, 'publishedDate_y'])[0:2] == '20':\n",
    "        book.loc[i, 'publishedCent'] = '21C'\n",
    "    else:\n",
    "        pass"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "==================================================="
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "merged_book = pd.read_csv('example.csv', encoding='euc-kr')\n",
    "merged_book.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "aaa = {str(v):k for k, v in author_dict.items()}\n",
    "merged_book['final_author_entity'] = [''] * len(merged_book)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i in tqdm(range(len(merged_book))):\n",
    "    author_list = str(merged_book.loc[i, 'authors']).split(',')\n",
    "    for aa in author_list:\n",
    "        # 딕셔너리의 value 중에서 탐색 -> 키 반환\n",
    "        try:\n",
    "            key = aaa.get(aa.strip())\n",
    "            if key is not None:\n",
    "                merged_book.loc[i, 'final_author_entity'] += (str(int(key)+1000)+', ')\n",
    "            else:\n",
    "                pass\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass\n",
    "    merged_book.loc[i, 'final_author_entity'].rstrip(', ')\n",
    "\n",
    "display(merged_book)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "merged_book[['authors', 'final_author_entity']]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "merged_book.columns"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "final_merged_book = merged_book.drop(columns=['Unnamed: 0', 'Unnamed: 0_x','Unnamed: 0.1', 'Column1',\n",
    "       'Unnamed: 0_y','author_entity_x'])\n",
    "final_merged_book"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "category_entity2 = {str(v):k for k, v in category_entity.items()}\n",
    "book['category_entity'] = [''] * len(book)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i in tqdm(range(len(book))):\n",
    "    categoryName = str(book.loc[i, 'categoryName'])\n",
    "    #entity_list = ''\n",
    "    try:\n",
    "        key = category_entity2.get(categoryName)\n",
    "        book.loc[i, 'category_entity'] += (str(int(key)))\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        pass\n",
    "\n",
    "display(book)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book2 = pd.read_csv('files/book.csv')\n",
    "max(book2['readerCount'])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book2[book2.duplicated(subset=['title','authors'],keep=False)==True].itemId.isna().value_counts()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# 푸아송 분포로 readercount 확인\n",
    "from scipy.stats import poisson\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "mu = 5\n",
    "readercount = book2['readerCount']\n",
    "pmf = poisson.pmf(readercount,mu)\n",
    "\n",
    "plt.bar(readercount, pmf)\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book2.to_csv('book_entity_1.csv', encoding='cp949')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "작가명이 한글자인 경우는 ENTITY 제외해야 함..... ex - 김, 가, 어쩌고 등"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## PickCount 분포 확인"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from scipy.stats import poisson\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "mu = 5\n",
    "readercount = book['readerCount']\n",
    "pmf = poisson.pmf(readercount,mu)\n",
    "\n",
    "plt.bar(readercount, pmf)\n",
    "plt.xlim(0,20)\n",
    "plt.show()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "0 : 0 ~ 3 적은 인기\n",
    "1 : 3 ~ 7 중간 인기\n",
    "2 : 7 ~ 10 적당한 인기\n",
    "3 : 많은 인기"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book['readerCount_entity'] = ['None'] * len(book)\n",
    "book['readerCount_entity']"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i in range(len(book)):\n",
    "    if not book.loc[i, 'readerCount'] or np.isnan(book.loc[i, 'readerCount']):\n",
    "        continue\n",
    "    print(book.loc[i, 'readerCount'])\n",
    "    if 0 <= int(book.loc[i, 'readerCount']) <= 3:\n",
    "        book.loc[i, 'readerCount_entity'] = 0\n",
    "    elif 3 < int(book.loc[i, 'readerCount']) <=7:\n",
    "        book.loc[i, 'readerCount_entity'] = 1\n",
    "    elif 7 < int(book.loc[i, 'readerCount']) <=10:\n",
    "        book.loc[i, 'readerCount_entity'] = 2\n",
    "    else:\n",
    "        book.loc[i, 'readerCount_entity'] = 3\n",
    "\n",
    "book"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "book = pd.read_csv('book_entity.csv',encoding='euc-kr', encoding_errors='ignore')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i, row in book.iterrows():\n",
    "    try:\n",
    "        new = 50 + int(row['readerCount_entity'])\n",
    "    except:\n",
    "        new = ''\n",
    "    book.at[i, 'readerCount_entity'] = new\n",
    "book"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book.to_csv('final_book_entity.csv', encoding='euc-kr')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "book[book['readerCount_entity']==53]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i, row in df.iterrows():\n",
    "    df.loc[i, 'category_entity'] = row['category_entity'] + 500\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i, row in df.iterrows():\n",
    "    if row['publishedCent']=='20C':\n",
    "        df.loc[i, 'publishedCent'] = 100\n",
    "    if row['publishedCent']=='21C':\n",
    "        df.loc[i, 'publishedCent'] = 101"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i, row in fb_df.iterrows():\n",
    "    if row['publishedCent']=='20C':\n",
    "        fb_df.loc[i, 'publishedCent'] = 100\n",
    "    if row['publishedCent']=='21C':\n",
    "        fb_df.loc[i, 'publishedCent'] = 101"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i, row in fb_df.iterrows():\n",
    "    fb_df.loc[i, 'category_entity'] = row['category_entity'] + 500"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i, row in fb_df.iterrows():\n",
    "    if fb_df.loc[i, 'readerCount'] ==1:\n",
    "        fb_df.loc[i, 'readerCount_entity'] = 50\n",
    "    elif 1 < fb_df.loc[i, 'readerCount'] <=15:\n",
    "        fb_df.loc[i, 'readerCount_entity'] = 51\n",
    "    elif fb_df.loc[i, 'readerCount'] > 16:\n",
    "        fb_df.loc[i, 'readerCount_entity'] = 52"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "fb_df.to_csv('final_book_entity.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 16자리 book id => int 매핑"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for i in range(len(fb_df[['id']])):\n",
    "    item_list = pd.concat([item_list, pd.DataFrame([{\"id\" : fb_df.loc[i, 'id'], \"remap_id\":i}])])\n",
    "item_list"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "item_list.to_csv('item_list.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}