Step19-CosineSimilarity_Word2Vec_GeneDisease.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<frozen importlib._bootstrap>:219: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vectors = Word2Vec.load(\"word2vec_for_gene.model\").wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_disease_abstract_map = pd.read_csv('gene_disease_abstract_map_with_stems.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cosine(row):\n",
    "    disease = row.disease\n",
    "    gene = row.gene\n",
    "    try:\n",
    "        cosine_similarity = word_vectors.similarity(disease, gene)\n",
    "    except:\n",
    "        try:\n",
    "            cosine_similarity = word_vectors.similarity(disease, gene.lower())\n",
    "        except:\n",
    "            cosine_similarity = 'UNKNOWN'\n",
    "    return cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_disease_abstract_map['cosine'] = gene_disease_abstract_map.apply(compute_cosine, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>doi</th>\n",
       "      <th>abstract</th>\n",
       "      <th>metaId</th>\n",
       "      <th>gene</th>\n",
       "      <th>disease</th>\n",
       "      <th>stems</th>\n",
       "      <th>cosine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [title, doi, abstract, metaId, gene, disease, stems, cosine]\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_disease_abstract_map[gene_disease_abstract_map.cosine == 'UNKNOWN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('test', 0.9999520182609558),\n",
       " (')', 0.9999470710754395),\n",
       " ('studi', 0.9999468326568604),\n",
       " ('infect', 0.9999461770057678),\n",
       " ('antibodi', 0.999944269657135),\n",
       " ('2', 0.9999435544013977),\n",
       " ('PRNT', 0.9999432563781738),\n",
       " ('revers', 0.9999416470527649),\n",
       " ('1', 0.9999412894248962),\n",
       " ('.', 0.9999408721923828)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_vectors.most_similar('COVID-19')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_disease_abstract_map.to_csv('gene_disease_abstract_map_with_cosine.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>doi</th>\n",
       "      <th>abstract</th>\n",
       "      <th>metaId</th>\n",
       "      <th>gene</th>\n",
       "      <th>disease</th>\n",
       "      <th>stems</th>\n",
       "      <th>cosine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>322</td>\n",
       "      <td>322</td>\n",
       "      <td>322</td>\n",
       "      <td>322.000000</td>\n",
       "      <td>322</td>\n",
       "      <td>322</td>\n",
       "      <td>322</td>\n",
       "      <td>322.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>192</td>\n",
       "      <td>192</td>\n",
       "      <td>192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>188</td>\n",
       "      <td>40</td>\n",
       "      <td>297</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>A combination of four serum miRNAs for screeni...</td>\n",
       "      <td>10.1007/s13577-020-00346-6</td>\n",
       "      <td>Serum microRNAs (miRNAs), with their noticeabl...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PRNT</td>\n",
       "      <td>cancer</td>\n",
       "      <td>['introduct', ':', 'the', 'express', 'specif',...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26</td>\n",
       "      <td>80</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>397598.782609</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>280807.382259</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>783.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.997417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>114664.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>543811.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>651015.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>764237.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.999943</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title  \\\n",
       "count                                                 322   \n",
       "unique                                                192   \n",
       "top     A combination of four serum miRNAs for screeni...   \n",
       "freq                                                   10   \n",
       "mean                                                  NaN   \n",
       "std                                                   NaN   \n",
       "min                                                   NaN   \n",
       "25%                                                   NaN   \n",
       "50%                                                   NaN   \n",
       "75%                                                   NaN   \n",
       "max                                                   NaN   \n",
       "\n",
       "                               doi  \\\n",
       "count                          322   \n",
       "unique                         192   \n",
       "top     10.1007/s13577-020-00346-6   \n",
       "freq                            10   \n",
       "mean                           NaN   \n",
       "std                            NaN   \n",
       "min                            NaN   \n",
       "25%                            NaN   \n",
       "50%                            NaN   \n",
       "75%                            NaN   \n",
       "max                            NaN   \n",
       "\n",
       "                                                 abstract         metaId  \\\n",
       "count                                                 322     322.000000   \n",
       "unique                                                192            NaN   \n",
       "top     Serum microRNAs (miRNAs), with their noticeabl...            NaN   \n",
       "freq                                                   10            NaN   \n",
       "mean                                                  NaN  397598.782609   \n",
       "std                                                   NaN  280807.382259   \n",
       "min                                                   NaN     783.000000   \n",
       "25%                                                   NaN  114664.000000   \n",
       "50%                                                   NaN  543811.000000   \n",
       "75%                                                   NaN  651015.000000   \n",
       "max                                                   NaN  764237.000000   \n",
       "\n",
       "        gene disease                                              stems  \\\n",
       "count    322     322                                                322   \n",
       "unique   188      40                                                297   \n",
       "top     PRNT  cancer  ['introduct', ':', 'the', 'express', 'specif',...   \n",
       "freq      26      80                                                  5   \n",
       "mean     NaN     NaN                                                NaN   \n",
       "std      NaN     NaN                                                NaN   \n",
       "min      NaN     NaN                                                NaN   \n",
       "25%      NaN     NaN                                                NaN   \n",
       "50%      NaN     NaN                                                NaN   \n",
       "75%      NaN     NaN                                                NaN   \n",
       "max      NaN     NaN                                                NaN   \n",
       "\n",
       "            cosine  \n",
       "count   322.000000  \n",
       "unique         NaN  \n",
       "top            NaN  \n",
       "freq           NaN  \n",
       "mean      0.999729  \n",
       "std       0.000244  \n",
       "min       0.997417  \n",
       "25%       0.999674  \n",
       "50%       0.999800  \n",
       "75%       0.999869  \n",
       "max       0.999943  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_disease_abstract_map.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dg_ext = pd.read_csv('../August/Dictionary/all_gene_disease_associations.tsv', sep='\\t', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geneId</th>\n",
       "      <th>geneSymbol</th>\n",
       "      <th>DSI</th>\n",
       "      <th>DPI</th>\n",
       "      <th>diseaseId</th>\n",
       "      <th>diseaseName</th>\n",
       "      <th>diseaseType</th>\n",
       "      <th>diseaseClass</th>\n",
       "      <th>diseaseSemanticType</th>\n",
       "      <th>score</th>\n",
       "      <th>EI</th>\n",
       "      <th>YearInitial</th>\n",
       "      <th>YearFinal</th>\n",
       "      <th>NofPmids</th>\n",
       "      <th>NofSnps</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.538</td>\n",
       "      <td>C0001418</td>\n",
       "      <td>Adenocarcinoma</td>\n",
       "      <td>group</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>LHGDN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.538</td>\n",
       "      <td>C0002736</td>\n",
       "      <td>Amyotrophic Lateral Sclerosis</td>\n",
       "      <td>disease</td>\n",
       "      <td>C18;C10</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.538</td>\n",
       "      <td>C0003578</td>\n",
       "      <td>Apnea</td>\n",
       "      <td>phenotype</td>\n",
       "      <td>C23;C08</td>\n",
       "      <td>Sign or Symptom</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.538</td>\n",
       "      <td>C0003864</td>\n",
       "      <td>Arthritis</td>\n",
       "      <td>disease</td>\n",
       "      <td>C05</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.538</td>\n",
       "      <td>C0008373</td>\n",
       "      <td>Cholesteatoma</td>\n",
       "      <td>disease</td>\n",
       "      <td>C17</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134937</th>\n",
       "      <td>115804232</td>\n",
       "      <td>CEROX1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C0005890</td>\n",
       "      <td>Body Height</td>\n",
       "      <td>phenotype</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Organism Attribute</td>\n",
       "      <td>0.10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>GWASCAT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134938</th>\n",
       "      <td>115891964</td>\n",
       "      <td>MIR223HG</td>\n",
       "      <td>0.861</td>\n",
       "      <td>0.077</td>\n",
       "      <td>C0023418</td>\n",
       "      <td>leukemia</td>\n",
       "      <td>disease</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134939</th>\n",
       "      <td>115891964</td>\n",
       "      <td>MIR223HG</td>\n",
       "      <td>0.861</td>\n",
       "      <td>0.077</td>\n",
       "      <td>C0023467</td>\n",
       "      <td>Leukemia, Myelocytic, Acute</td>\n",
       "      <td>disease</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134940</th>\n",
       "      <td>115891964</td>\n",
       "      <td>MIR223HG</td>\n",
       "      <td>0.861</td>\n",
       "      <td>0.077</td>\n",
       "      <td>C0598766</td>\n",
       "      <td>Leukemogenesis</td>\n",
       "      <td>disease</td>\n",
       "      <td>C23;C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134941</th>\n",
       "      <td>115891964</td>\n",
       "      <td>MIR223HG</td>\n",
       "      <td>0.861</td>\n",
       "      <td>0.077</td>\n",
       "      <td>C1332977</td>\n",
       "      <td>Childhood Leukemia</td>\n",
       "      <td>disease</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1134942 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            geneId geneSymbol    DSI    DPI diseaseId  \\\n",
       "0                1       A1BG  0.700  0.538  C0001418   \n",
       "1                1       A1BG  0.700  0.538  C0002736   \n",
       "2                1       A1BG  0.700  0.538  C0003578   \n",
       "3                1       A1BG  0.700  0.538  C0003864   \n",
       "4                1       A1BG  0.700  0.538  C0008373   \n",
       "...            ...        ...    ...    ...       ...   \n",
       "1134937  115804232     CEROX1    NaN    NaN  C0005890   \n",
       "1134938  115891964   MIR223HG  0.861  0.077  C0023418   \n",
       "1134939  115891964   MIR223HG  0.861  0.077  C0023467   \n",
       "1134940  115891964   MIR223HG  0.861  0.077  C0598766   \n",
       "1134941  115891964   MIR223HG  0.861  0.077  C1332977   \n",
       "\n",
       "                           diseaseName diseaseType diseaseClass  \\\n",
       "0                       Adenocarcinoma       group          C04   \n",
       "1        Amyotrophic Lateral Sclerosis     disease      C18;C10   \n",
       "2                                Apnea   phenotype      C23;C08   \n",
       "3                            Arthritis     disease          C05   \n",
       "4                        Cholesteatoma     disease          C17   \n",
       "...                                ...         ...          ...   \n",
       "1134937                    Body Height   phenotype          NaN   \n",
       "1134938                       leukemia     disease          C04   \n",
       "1134939    Leukemia, Myelocytic, Acute     disease          C04   \n",
       "1134940                 Leukemogenesis     disease      C23;C04   \n",
       "1134941             Childhood Leukemia     disease          C04   \n",
       "\n",
       "         diseaseSemanticType  score   EI  YearInitial  YearFinal  NofPmids  \\\n",
       "0         Neoplastic Process   0.01  1.0       2008.0     2008.0         1   \n",
       "1        Disease or Syndrome   0.01  1.0       2008.0     2008.0         1   \n",
       "2            Sign or Symptom   0.01  1.0       2017.0     2017.0         1   \n",
       "3        Disease or Syndrome   0.01  1.0       2019.0     2019.0         1   \n",
       "4        Disease or Syndrome   0.01  1.0       2020.0     2020.0         1   \n",
       "...                      ...    ...  ...          ...        ...       ...   \n",
       "1134937   Organism Attribute   0.10  1.0       2019.0     2019.0         1   \n",
       "1134938   Neoplastic Process   0.01  1.0       2016.0     2016.0         1   \n",
       "1134939   Neoplastic Process   0.01  1.0       2016.0     2016.0         1   \n",
       "1134940   Neoplastic Process   0.01  1.0       2016.0     2016.0         1   \n",
       "1134941   Neoplastic Process   0.01  1.0       2016.0     2016.0         1   \n",
       "\n",
       "         NofSnps   source  \n",
       "0              0    LHGDN  \n",
       "1              0   BEFREE  \n",
       "2              0   BEFREE  \n",
       "3              0   BEFREE  \n",
       "4              0   BEFREE  \n",
       "...          ...      ...  \n",
       "1134937        0  GWASCAT  \n",
       "1134938        0   BEFREE  \n",
       "1134939        0   BEFREE  \n",
       "1134940        0   BEFREE  \n",
       "1134941        0   BEFREE  \n",
       "\n",
       "[1134942 rows x 16 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dg_ext.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dg_cosine = gene_disease_abstract_map[['disease', 'gene', 'cosine', 'title', 'abstract', 'doi']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dg_ext['disease'] = dg_ext.diseaseName.str.lower()\n",
    "dg_ext['gene'] = dg_ext.geneSymbol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df = dg_ext.merge(dg_cosine, on=['disease', 'gene'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geneId</th>\n",
       "      <th>geneSymbol</th>\n",
       "      <th>DSI</th>\n",
       "      <th>DPI</th>\n",
       "      <th>diseaseId</th>\n",
       "      <th>diseaseName</th>\n",
       "      <th>diseaseType</th>\n",
       "      <th>diseaseClass</th>\n",
       "      <th>diseaseSemanticType</th>\n",
       "      <th>score</th>\n",
       "      <th>...</th>\n",
       "      <th>YearFinal</th>\n",
       "      <th>NofPmids</th>\n",
       "      <th>NofSnps</th>\n",
       "      <th>source</th>\n",
       "      <th>disease</th>\n",
       "      <th>gene</th>\n",
       "      <th>cosine</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>doi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7012</td>\n",
       "      <td>TERC</td>\n",
       "      <td>0.471</td>\n",
       "      <td>0.846</td>\n",
       "      <td>C0002871</td>\n",
       "      <td>Anemia</td>\n",
       "      <td>disease</td>\n",
       "      <td>C15</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>0.10</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>HPO</td>\n",
       "      <td>anemia</td>\n",
       "      <td>TERC</td>\n",
       "      <td>0.999758</td>\n",
       "      <td>A Case Series of TERC Variant Telomere Biology...</td>\n",
       "      <td>TERC variant telomere biology disorders (TBDs)...</td>\n",
       "      <td>10.14740/jh826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8847</td>\n",
       "      <td>DLEU2</td>\n",
       "      <td>0.650</td>\n",
       "      <td>0.500</td>\n",
       "      <td>C0023418</td>\n",
       "      <td>leukemia</td>\n",
       "      <td>disease</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.03</td>\n",
       "      <td>...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "      <td>leukemia</td>\n",
       "      <td>DLEU2</td>\n",
       "      <td>0.999760</td>\n",
       "      <td>Genomic alterations in chronic lymphocytic leu...</td>\n",
       "      <td>Background Chronic lymphocytic leukemia (CLL) ...</td>\n",
       "      <td>10.5045/br.2020.2020080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10866</td>\n",
       "      <td>HCP5</td>\n",
       "      <td>0.595</td>\n",
       "      <td>0.808</td>\n",
       "      <td>C0025202</td>\n",
       "      <td>melanoma</td>\n",
       "      <td>disease</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "      <td>melanoma</td>\n",
       "      <td>HCP5</td>\n",
       "      <td>0.999778</td>\n",
       "      <td>A Nomogram Combining a Four-Gene Biomarker and...</td>\n",
       "      <td>BACKGROUND: Currently there is no effective pr...</td>\n",
       "      <td>10.3389/fonc.2021.593587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60674</td>\n",
       "      <td>GAS5</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.808</td>\n",
       "      <td>C0029456</td>\n",
       "      <td>Osteoporosis</td>\n",
       "      <td>disease</td>\n",
       "      <td>C18;C05</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "      <td>osteoporosis</td>\n",
       "      <td>GAS5</td>\n",
       "      <td>0.999634</td>\n",
       "      <td>lncRNA GAS5 Is Upregulated in Osteoporosis and...</td>\n",
       "      <td>Background It has been reported that lncRNA gr...</td>\n",
       "      <td>10.2147/cia.s235197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>283131</td>\n",
       "      <td>NEAT1</td>\n",
       "      <td>0.502</td>\n",
       "      <td>0.769</td>\n",
       "      <td>C0001418</td>\n",
       "      <td>Adenocarcinoma</td>\n",
       "      <td>group</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "      <td>adenocarcinoma</td>\n",
       "      <td>NEAT1</td>\n",
       "      <td>0.999818</td>\n",
       "      <td>Detection of the long noncoding RNAs nuclear-e...</td>\n",
       "      <td>OBJECTIVES Long noncoding RNAs (lncRNAs) in HI...</td>\n",
       "      <td>10.1111/hiv.12276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>378938</td>\n",
       "      <td>MALAT1</td>\n",
       "      <td>0.435</td>\n",
       "      <td>0.846</td>\n",
       "      <td>C0007097</td>\n",
       "      <td>Carcinoma</td>\n",
       "      <td>group</td>\n",
       "      <td>C04</td>\n",
       "      <td>Neoplastic Process</td>\n",
       "      <td>0.10</td>\n",
       "      <td>...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>BEFREE</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>MALAT1</td>\n",
       "      <td>0.999829</td>\n",
       "      <td>Loss of the abundant nuclear non-coding RNA MA...</td>\n",
       "      <td>The metastasis-associated lung adenocarcinoma ...</td>\n",
       "      <td>10.4161/rna.21089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>100048912</td>\n",
       "      <td>CDKN2B-AS1</td>\n",
       "      <td>0.491</td>\n",
       "      <td>0.769</td>\n",
       "      <td>C0017601</td>\n",
       "      <td>Glaucoma</td>\n",
       "      <td>disease</td>\n",
       "      <td>C11</td>\n",
       "      <td>Disease or Syndrome</td>\n",
       "      <td>0.15</td>\n",
       "      <td>...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>BEFREE;GWASCAT;GWASDB</td>\n",
       "      <td>glaucoma</td>\n",
       "      <td>CDKN2B-AS1</td>\n",
       "      <td>0.999848</td>\n",
       "      <td>Common Variants in CDKN2B-AS1 Associated with ...</td>\n",
       "      <td>BACKGROUND: To date, only a small portion of t...</td>\n",
       "      <td>10.1371/journal.pone.0033389</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      geneId  geneSymbol    DSI    DPI diseaseId     diseaseName diseaseType  \\\n",
       "0       7012        TERC  0.471  0.846  C0002871          Anemia     disease   \n",
       "1       8847       DLEU2  0.650  0.500  C0023418        leukemia     disease   \n",
       "2      10866        HCP5  0.595  0.808  C0025202        melanoma     disease   \n",
       "3      60674        GAS5  0.497  0.808  C0029456    Osteoporosis     disease   \n",
       "4     283131       NEAT1  0.502  0.769  C0001418  Adenocarcinoma       group   \n",
       "5     378938      MALAT1  0.435  0.846  C0007097       Carcinoma       group   \n",
       "6  100048912  CDKN2B-AS1  0.491  0.769  C0017601        Glaucoma     disease   \n",
       "\n",
       "  diseaseClass  diseaseSemanticType  score  ...  YearFinal  NofPmids  NofSnps  \\\n",
       "0          C15  Disease or Syndrome   0.10  ...        NaN         0        0   \n",
       "1          C04   Neoplastic Process   0.03  ...     2014.0         3        0   \n",
       "2          C04   Neoplastic Process   0.01  ...     2019.0         1        0   \n",
       "3      C18;C05  Disease or Syndrome   0.01  ...     2019.0         1        0   \n",
       "4          C04   Neoplastic Process   0.01  ...     2016.0         1        0   \n",
       "5          C04   Neoplastic Process   0.10  ...     2019.0        12        0   \n",
       "6          C11  Disease or Syndrome   0.15  ...     2019.0         9       13   \n",
       "\n",
       "                  source         disease        gene    cosine  \\\n",
       "0                    HPO          anemia        TERC  0.999758   \n",
       "1                 BEFREE        leukemia       DLEU2  0.999760   \n",
       "2                 BEFREE        melanoma        HCP5  0.999778   \n",
       "3                 BEFREE    osteoporosis        GAS5  0.999634   \n",
       "4                 BEFREE  adenocarcinoma       NEAT1  0.999818   \n",
       "5                 BEFREE       carcinoma      MALAT1  0.999829   \n",
       "6  BEFREE;GWASCAT;GWASDB        glaucoma  CDKN2B-AS1  0.999848   \n",
       "\n",
       "                                               title  \\\n",
       "0  A Case Series of TERC Variant Telomere Biology...   \n",
       "1  Genomic alterations in chronic lymphocytic leu...   \n",
       "2  A Nomogram Combining a Four-Gene Biomarker and...   \n",
       "3  lncRNA GAS5 Is Upregulated in Osteoporosis and...   \n",
       "4  Detection of the long noncoding RNAs nuclear-e...   \n",
       "5  Loss of the abundant nuclear non-coding RNA MA...   \n",
       "6  Common Variants in CDKN2B-AS1 Associated with ...   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  TERC variant telomere biology disorders (TBDs)...   \n",
       "1  Background Chronic lymphocytic leukemia (CLL) ...   \n",
       "2  BACKGROUND: Currently there is no effective pr...   \n",
       "3  Background It has been reported that lncRNA gr...   \n",
       "4  OBJECTIVES Long noncoding RNAs (lncRNAs) in HI...   \n",
       "5  The metastasis-associated lung adenocarcinoma ...   \n",
       "6  BACKGROUND: To date, only a small portion of t...   \n",
       "\n",
       "                            doi  \n",
       "0                10.14740/jh826  \n",
       "1       10.5045/br.2020.2020080  \n",
       "2      10.3389/fonc.2021.593587  \n",
       "3           10.2147/cia.s235197  \n",
       "4             10.1111/hiv.12276  \n",
       "5             10.4161/rna.21089  \n",
       "6  10.1371/journal.pone.0033389  \n",
       "\n",
       "[7 rows x 22 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_983022/2387617006.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  dg_cosine['found_flag'] = 0\n"
     ]
    }
   ],
   "source": [
    "dg_cosine['found_flag'] = 0\n",
    "merged_df['found_flag'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cosine_similarity_known_list = merged_df['cosine']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "cosine_similarity_known_list = [float(x) for x in cosine_similarity_known_list] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_cosine_known = sum(cosine_similarity_known_list) / len(cosine_similarity_known_list)\n",
    "max_cosine_known = max(cosine_similarity_known_list)\n",
    "min_cosine_known = min(cosine_similarity_known_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min known cosine:\n",
      "0.9996343851089478\n",
      "avg known cosine:\n",
      "0.9997752904891968\n",
      "max known cosine:\n",
      "0.999848484992981\n"
     ]
    }
   ],
   "source": [
    "print(\"min known cosine:\")\n",
    "print(min_cosine_known)\n",
    "print(\"avg known cosine:\")\n",
    "print(avg_cosine_known)\n",
    "print(\"max known cosine:\")\n",
    "print(max_cosine_known)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_df = dg_cosine.append(merged_df[['disease', 'gene', 'cosine', 'found_flag', 'title', 'abstract', 'doi']], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_df = main_df.sort_values('found_flag', ascending=False).drop_duplicates(subset=['disease', 'gene'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>disease</th>\n",
       "      <th>gene</th>\n",
       "      <th>cosine</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>doi</th>\n",
       "      <th>found_flag</th>\n",
       "      <th>association</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>glaucoma</td>\n",
       "      <td>CDKN2B-AS1</td>\n",
       "      <td>0.999848</td>\n",
       "      <td>Common Variants in CDKN2B-AS1 Associated with ...</td>\n",
       "      <td>BACKGROUND: To date, only a small portion of t...</td>\n",
       "      <td>10.1371/journal.pone.0033389</td>\n",
       "      <td>1</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>carcinoma</td>\n",
       "      <td>MALAT1</td>\n",
       "      <td>0.999829</td>\n",
       "      <td>Loss of the abundant nuclear non-coding RNA MA...</td>\n",
       "      <td>The metastasis-associated lung adenocarcinoma ...</td>\n",
       "      <td>10.4161/rna.21089</td>\n",
       "      <td>1</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>adenocarcinoma</td>\n",
       "      <td>NEAT1</td>\n",
       "      <td>0.999818</td>\n",
       "      <td>Detection of the long noncoding RNAs nuclear-e...</td>\n",
       "      <td>OBJECTIVES Long noncoding RNAs (lncRNAs) in HI...</td>\n",
       "      <td>10.1111/hiv.12276</td>\n",
       "      <td>1</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325</th>\n",
       "      <td>osteoporosis</td>\n",
       "      <td>GAS5</td>\n",
       "      <td>0.999634</td>\n",
       "      <td>lncRNA GAS5 Is Upregulated in Osteoporosis and...</td>\n",
       "      <td>Background It has been reported that lncRNA gr...</td>\n",
       "      <td>10.2147/cia.s235197</td>\n",
       "      <td>1</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>324</th>\n",
       "      <td>melanoma</td>\n",
       "      <td>HCP5</td>\n",
       "      <td>0.999778</td>\n",
       "      <td>A Nomogram Combining a Four-Gene Biomarker and...</td>\n",
       "      <td>BACKGROUND: Currently there is no effective pr...</td>\n",
       "      <td>10.3389/fonc.2021.593587</td>\n",
       "      <td>1</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>cancer</td>\n",
       "      <td>miR-125a-5p</td>\n",
       "      <td>0.999750</td>\n",
       "      <td>Upregulation of TRIAP1 by the lncRNA MFI2-AS1/...</td>\n",
       "      <td>Background Thyroid cancer is a very common end...</td>\n",
       "      <td>10.2147/ott.s236476</td>\n",
       "      <td>0</td>\n",
       "      <td>Medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>cancer</td>\n",
       "      <td>miR-148b-3p</td>\n",
       "      <td>0.999824</td>\n",
       "      <td>Downregulated exosomal microRNA-148b-3p in can...</td>\n",
       "      <td>OBJECTIVE Exosomes derived from cancer-associa...</td>\n",
       "      <td>10.1007/s13402-020-00500-0</td>\n",
       "      <td>0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>cancer</td>\n",
       "      <td>RFPL3S</td>\n",
       "      <td>0.999853</td>\n",
       "      <td>Long non-coding RNA RFPL3S is a novel prognost...</td>\n",
       "      <td>Long non-coding RNAs (lncRNAs) are functional ...</td>\n",
       "      <td>10.3892/ol.2020.11642</td>\n",
       "      <td>0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>syndrome</td>\n",
       "      <td>MEG3</td>\n",
       "      <td>0.999783</td>\n",
       "      <td>The value of circulating long non-coding RNA m...</td>\n",
       "      <td>OBJECTIVE This study was to evaluate the poten...</td>\n",
       "      <td>10.1002/jcla.23488</td>\n",
       "      <td>0</td>\n",
       "      <td>Medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>miR-145-5p</td>\n",
       "      <td>0.999857</td>\n",
       "      <td>Decreased inhibition of exosomal miRNAs on SAR...</td>\n",
       "      <td>Elderly people and patients with comorbidities...</td>\n",
       "      <td>10.1038/s41392-021-00716-y</td>\n",
       "      <td>0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>277 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            disease         gene    cosine  \\\n",
       "328        glaucoma   CDKN2B-AS1  0.999848   \n",
       "327       carcinoma       MALAT1  0.999829   \n",
       "326  adenocarcinoma        NEAT1  0.999818   \n",
       "325    osteoporosis         GAS5  0.999634   \n",
       "324        melanoma         HCP5  0.999778   \n",
       "..              ...          ...       ...   \n",
       "109          cancer  miR-125a-5p  0.999750   \n",
       "108          cancer  miR-148b-3p  0.999824   \n",
       "107          cancer       RFPL3S  0.999853   \n",
       "106        syndrome         MEG3  0.999783   \n",
       "164        COVID-19   miR-145-5p  0.999857   \n",
       "\n",
       "                                                 title  \\\n",
       "328  Common Variants in CDKN2B-AS1 Associated with ...   \n",
       "327  Loss of the abundant nuclear non-coding RNA MA...   \n",
       "326  Detection of the long noncoding RNAs nuclear-e...   \n",
       "325  lncRNA GAS5 Is Upregulated in Osteoporosis and...   \n",
       "324  A Nomogram Combining a Four-Gene Biomarker and...   \n",
       "..                                                 ...   \n",
       "109  Upregulation of TRIAP1 by the lncRNA MFI2-AS1/...   \n",
       "108  Downregulated exosomal microRNA-148b-3p in can...   \n",
       "107  Long non-coding RNA RFPL3S is a novel prognost...   \n",
       "106  The value of circulating long non-coding RNA m...   \n",
       "164  Decreased inhibition of exosomal miRNAs on SAR...   \n",
       "\n",
       "                                              abstract  \\\n",
       "328  BACKGROUND: To date, only a small portion of t...   \n",
       "327  The metastasis-associated lung adenocarcinoma ...   \n",
       "326  OBJECTIVES Long noncoding RNAs (lncRNAs) in HI...   \n",
       "325  Background It has been reported that lncRNA gr...   \n",
       "324  BACKGROUND: Currently there is no effective pr...   \n",
       "..                                                 ...   \n",
       "109  Background Thyroid cancer is a very common end...   \n",
       "108  OBJECTIVE Exosomes derived from cancer-associa...   \n",
       "107  Long non-coding RNAs (lncRNAs) are functional ...   \n",
       "106  OBJECTIVE This study was to evaluate the poten...   \n",
       "164  Elderly people and patients with comorbidities...   \n",
       "\n",
       "                              doi  found_flag association  \n",
       "328  10.1371/journal.pone.0033389           1    Verified  \n",
       "327             10.4161/rna.21089           1    Verified  \n",
       "326             10.1111/hiv.12276           1    Verified  \n",
       "325           10.2147/cia.s235197           1    Verified  \n",
       "324      10.3389/fonc.2021.593587           1    Verified  \n",
       "..                            ...         ...         ...  \n",
       "109           10.2147/ott.s236476           0      Medium  \n",
       "108    10.1007/s13402-020-00500-0           0        High  \n",
       "107         10.3892/ol.2020.11642           0        High  \n",
       "106            10.1002/jcla.23488           0      Medium  \n",
       "164    10.1038/s41392-021-00716-y           0        High  \n",
       "\n",
       "[277 rows x 8 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_association(row):\n",
    "    association_type = 'ERROR'\n",
    "    found_flag = row['found_flag']\n",
    "    if found_flag == 1:\n",
    "        association_type = 'Verified'\n",
    "    else:\n",
    "        cosine = row['cosine']\n",
    "        if cosine == 'UNKNOWN':\n",
    "            association_type = 'Low'\n",
    "        else:\n",
    "            cosine = float(cosine)\n",
    "            distance_from_max_known = abs(max_cosine_known - cosine)\n",
    "            distance_from_avg_known = abs(avg_cosine_known - cosine)\n",
    "            distance_from_min_known = abs(min_cosine_known - cosine)\n",
    "            min_distance = min(distance_from_max_known, distance_from_avg_known, distance_from_min_known)\n",
    "            if min_distance == distance_from_max_known:\n",
    "                association_type = 'High'\n",
    "            elif min_distance == distance_from_avg_known:\n",
    "                association_type = 'Medium'\n",
    "            elif min_distance == distance_from_min_known:\n",
    "                association_type = 'Low'\n",
    "    return association_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_df['association'] = main_df.apply(compute_association, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_df.to_csv(\"gene_disease_association_with_confidence.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}