Untitled3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] Malachi Wideman LOSE YOUR MIND @ mightymal13 https : / / t. co / ZX6Pq6V2w8 [SEP]'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(['101', '58335', '12806', '21660', '10589', '149', '21793', '11259', '100061', '11273', '75416', '58521', '137', '20970', '16889', '10415', '45389', '14120', '131', '120', '120', '188', '119', '11170', '120', '163', '12674', '11211', '11127', '11703', '11211', '11779', '10729', '10874', '11396', '102'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download to another location: https://huggingface.co/transformers/model_doc/bert.html#transformers.BertModel.get_input_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BertModel.from_pretrained('/dev/bert/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor([101, 58335, 12806, 21660, 10589, 149, 21793, 11259, 100061, 11273, 75416, 58521, 137, 20970, 16889, 10415, 45389, 14120, 131, 120, 120, 188, 119, 11170, 120, 163, 12674, 11211, 11127, 11703, 11211, 11779, 10729, 10874, 11396, 102]).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids)\n",
    "last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\")).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids)\n",
    "last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-7.4056e-02, -2.5629e-01,  3.9576e-03,  ...,  3.3745e-01,\n",
       "           9.6348e-02, -7.4602e-02],\n",
       "         [-3.7866e-01,  2.4644e-04,  3.8359e-01,  ...,  4.0054e-01,\n",
       "          -3.5663e-01, -3.5926e-01],\n",
       "         [-3.2465e-01, -6.2971e-01,  6.9400e-01,  ..., -1.2656e-01,\n",
       "           1.1308e-01, -3.0379e-01],\n",
       "         ...,\n",
       "         [-1.4325e-01, -4.6731e-01,  8.1009e-01,  ...,  3.6486e-01,\n",
       "           4.4799e-01, -1.8962e-01],\n",
       "         [-2.4273e-01, -6.3758e-01,  1.4377e-01,  ...,  1.1618e+00,\n",
       "           7.6320e-01, -1.2245e-01],\n",
       "         [ 1.0538e-01, -6.8167e-01,  5.1111e-01,  ...,  3.6763e-01,\n",
       "           3.9284e-01, -3.1134e-02]]], grad_fn=<NativeLayerNormBackward>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 31178, 117, 15127, 17835, 10124, 21610, 10112, 102]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(\"Hello, my dog is cute\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 36, 768])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 31178, 117, 15127, 17835, 10124, 21610, 10112, 102]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(\"Hello, my dog is cute\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] Hello, my dog is cute'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode([101, 31178, 117, 15127, 17835, 10124, 21610, 10112])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "str = '[CLS] Malachi'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 101, 58335, 12806, 102]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]', '[CLS]', 'Mala', '##chi', '[SEP]']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens([101, 101, 58335, 12806, 102])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor([101, 101, 58335, 12806, 102]).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids)\n",
    "last_hidden_states = outputs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 1.2833e-01, -8.7672e-02,  5.1223e-01,  ...,  5.1597e-02,\n",
       "           4.0623e-02,  3.8689e-02],\n",
       "         [ 1.3011e-01, -8.9307e-02,  5.1085e-01,  ...,  5.1296e-02,\n",
       "           4.0854e-02,  3.8339e-02],\n",
       "         [-2.2348e-01, -1.6424e-01,  7.4471e-01,  ...,  1.6253e-01,\n",
       "          -2.9908e-01,  3.1246e-01],\n",
       "         [-6.6516e-04, -3.8011e-01,  6.1914e-01,  ..., -3.3696e-01,\n",
       "          -7.9211e-02, -7.4923e-02],\n",
       "         [ 2.4696e-01, -3.8488e-01,  1.0297e+00,  ..., -1.6962e-01,\n",
       "          -5.3971e-02,  3.5280e-01]]], grad_fn=<NativeLayerNormBackward>)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 5, 768])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}