bert_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_tokens = pd.read_csv('s3://recsys_challenge_2020/user_tokens.csv')\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)\n",
    "model = BertModel.from_pretrained('/dev/bert/', configuration)\n",
    "input_ids = torch.tensor([101, 108, 31278, 90939, 70325, 196, 199, 71436, 10107, 29190, 10107, 106, 16680, 68314, 10153, 17015, 15934, 10104, 108, 10233, 12396, 14945, 10107, 10858, 11405, 13600, 13597, 169, 57343, 64482, 119, 119, 119, 100, 11741, 16381, 10109, 68830, 10110, 20886, 108, 10233, 11127, 21768, 100, 14120, 131, 120, 120, 188, 119, 11170, 120, 12132, 10884, 10157, 11490, 12022, 10113, 10731, 10729, 11565, 14120, 131, 120, 120, 188, 119, 11170, 120, 162, 11211, 11703, 12022, 11211, 10240, 44466, 100886, 102]).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids)\n",
    "last_hidden_states = outputs[0][0][0]  # The last hidden-state is the first element of the output tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] # ENVIVO | ¡ Buenas noches! Comienza una nueva edición de # 24HorasCentral junto a Iván Núñez... [UNK] Sigue la señal en vivo # 24Play [UNK] https : / / t. co / mi3yDUaS2F https : / / t. co / Y6qU6gwdCL [SEP]'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode([101, 108, 31278, 90939, 70325, 196, 199, 71436, 10107, 29190, 10107, 106, 16680, 68314, 10153, 17015, 15934, 10104, 108, 10233, 12396, 14945, 10107, 10858, 11405, 13600, 13597, 169, 57343, 64482, 119, 119, 119, 100, 11741, 16381, 10109, 68830, 10110, 20886, 108, 10233, 11127, 21768, 100, 14120, 131, 120, 120, 188, 119, 11170, 120, 12132, 10884, 10157, 11490, 12022, 10113, 10731, 10729, 11565, 14120, 131, 120, 120, 188, 119, 11170, 120, 162, 11211, 11703, 12022, 11211, 10240, 44466, 100886, 102])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-3.1164e-01, -4.9567e-02, -1.8035e-01,  2.7875e-02,  9.6286e-02,\n",
       "        -1.1317e-01,  3.8975e-01,  5.9788e-02, -1.8645e-01, -2.5343e-01,\n",
       "         1.2640e-01,  1.8534e-01,  3.6691e-01,  2.5003e-01, -7.2783e-01,\n",
       "         2.4973e-02,  1.1377e-02,  3.5736e-01,  1.5089e-01,  8.0563e-02,\n",
       "         9.2578e-03,  1.8207e-01, -1.3513e-01, -1.5369e-02, -2.0341e-01,\n",
       "        -2.1747e-01,  1.8431e-02,  4.8761e-01,  5.4433e-01,  2.9416e-01,\n",
       "         3.3940e-01,  1.9408e-01, -1.2509e-01,  9.5964e-02,  1.4272e-01,\n",
       "        -2.6225e-01, -1.8296e+00,  1.1323e-02, -1.1208e-01, -3.4605e-02,\n",
       "         4.4772e-02,  2.0283e-01, -1.3155e-01, -1.3792e-01,  4.2419e-02,\n",
       "         1.2304e+00,  2.0679e-01,  3.3012e-01,  1.1707e+00,  7.9489e-02,\n",
       "        -1.4379e-01, -5.4192e-01, -2.1640e-01, -1.4817e+00, -4.4211e-02,\n",
       "         2.4996e-01,  1.5661e-02,  6.0621e-02, -1.8868e-01, -1.8413e-01,\n",
       "         7.2964e-03, -3.6828e-02,  2.6129e-01,  7.8231e-03,  2.2604e-01,\n",
       "         5.6580e-02, -1.7340e-01, -7.0535e-02, -2.3081e-01,  2.9581e-02,\n",
       "         1.8907e-01, -7.7927e-03, -3.4463e-02, -3.3769e-01,  6.2085e-02,\n",
       "         6.6017e-03,  5.3529e-02,  1.4392e-01, -6.7354e-01,  5.8455e-01,\n",
       "        -3.3202e-01, -1.7781e-01, -3.8297e-01,  2.5050e-01, -1.5817e-01,\n",
       "        -1.5637e-01, -1.3972e-01,  8.8274e-03, -3.1193e-01, -5.2330e-03,\n",
       "        -3.1015e-01, -1.7894e-01, -3.7934e-01, -3.3872e-02,  3.6230e-02,\n",
       "        -5.5094e-01,  3.0827e-02,  6.0444e-02,  2.1285e-01,  3.4410e-02,\n",
       "        -2.0747e-01,  1.4624e+00, -9.6516e-02, -2.0285e-01, -2.8758e+00,\n",
       "         7.1685e-02, -2.0123e-01, -3.0936e-01,  5.3199e-01,  1.2230e-01,\n",
       "         8.1956e-02, -2.7731e-01,  5.2838e-01, -2.0401e-01, -1.7810e-01,\n",
       "         2.7027e-01,  7.1271e-02, -3.3320e-01, -1.9835e-01, -3.9524e-01,\n",
       "        -1.0847e-01, -3.1769e-01, -1.3851e-02, -3.0103e-01, -2.5885e-01,\n",
       "        -3.7008e-01,  3.8289e-01, -2.4122e-01, -5.2675e-02,  8.0350e-01,\n",
       "         2.6473e-01,  1.7537e+00,  1.1817e-01,  9.0549e-02, -2.4417e-01,\n",
       "         1.6117e+00,  1.1194e-01,  1.3509e-01,  3.0544e-01,  2.5672e-01,\n",
       "        -1.7637e-01, -1.1499e-01, -3.6139e-01, -4.1337e-02, -2.3232e-01,\n",
       "         1.0171e-01, -2.3256e-01,  1.5083e-01, -1.3468e-01, -1.3573e-01,\n",
       "         1.6173e-02, -2.2671e-01,  7.6579e-02, -2.5343e-01, -1.2679e-01,\n",
       "        -1.0918e-01,  3.2601e-01, -3.2303e-01,  7.0725e-02,  5.1936e-01,\n",
       "         1.4984e+00,  2.8422e-01, -6.6885e-02,  1.9571e-01, -1.8713e-01,\n",
       "        -2.5129e-01, -3.8271e-01, -2.7025e-01,  2.5741e-02, -5.5135e-01,\n",
       "        -1.5479e-01, -7.9235e-02, -1.3692e-01,  6.6706e-02,  1.8734e-01,\n",
       "        -3.0018e-01, -1.9811e-01, -1.5047e-02,  3.0172e-02,  9.2474e-02,\n",
       "         2.6625e-02,  4.3988e-01, -2.0532e-01, -5.4617e-02, -3.1088e-02,\n",
       "        -1.0670e-01, -1.3672e-01, -3.2521e-01, -1.5645e-01, -1.7185e-01,\n",
       "         1.3517e-01,  2.6701e-02,  5.1355e-02,  2.4779e-01,  2.2525e-01,\n",
       "         2.1217e-01, -5.0566e-01,  1.6557e-01, -2.6629e-01,  5.3224e-02,\n",
       "         4.9084e-01,  1.7466e-01, -6.7205e-02, -1.3988e-01, -6.4878e-02,\n",
       "         5.8804e-01, -8.7675e-02, -8.4234e-02,  6.4814e-02,  6.9239e-02,\n",
       "         5.9876e-02,  8.5848e-02,  9.5514e-01, -8.9422e-02,  4.8883e-01,\n",
       "         1.3341e-01,  8.3476e-02, -4.8884e-02,  1.6384e-01, -2.1105e+00,\n",
       "        -2.6924e-01, -2.7174e-01,  5.2659e-01,  3.4364e-01, -1.6585e-02,\n",
       "         9.6607e-02, -4.3533e-01,  3.0182e-01,  8.8105e-03, -1.8271e-01,\n",
       "         9.9709e-02, -1.6784e-01,  1.2595e-01, -1.4539e+00,  5.4950e-02,\n",
       "         1.2727e-01,  1.7506e-01, -1.4045e-01, -1.7878e-01, -1.8425e-01,\n",
       "         1.6989e-01,  2.4157e-01,  1.2302e-02, -2.1150e-01,  3.2068e-01,\n",
       "        -2.3067e-01,  1.2051e+00,  6.6406e-02,  4.1650e-01, -9.4860e-03,\n",
       "         4.4990e-02,  7.8595e-02,  5.8286e-01, -5.2871e-02, -3.3147e-01,\n",
       "         9.7372e-02, -6.5911e-03,  2.6156e-01,  2.1119e-01, -4.0019e-01,\n",
       "        -5.6206e-02, -1.0966e-02,  1.1725e-01, -2.2355e-01, -1.2097e-01,\n",
       "         4.6912e-02, -4.5033e-02,  3.0396e-01,  1.0580e-01, -4.2669e-01,\n",
       "         2.0407e-01,  1.8600e-01,  8.8257e-02,  2.6991e-01, -5.0582e-03,\n",
       "         1.8494e-01, -3.1388e-01, -2.0047e-01, -1.4389e-01,  2.3031e-01,\n",
       "         2.0685e-02,  1.9062e-01,  4.4168e-01, -5.2698e-02,  7.1185e-02,\n",
       "         1.2879e-01, -1.1270e-01, -1.4037e-02, -4.2215e-01, -8.9047e-01,\n",
       "        -1.6354e-01, -3.1157e-03,  3.3752e-01, -4.2926e-01, -2.4030e-01,\n",
       "        -2.9406e-01, -2.5910e-03, -1.3362e-01, -1.3307e-01, -1.0107e-01,\n",
       "        -1.3528e-01, -5.0325e-01,  1.2698e-01,  2.1275e-01, -1.1756e+00,\n",
       "        -1.3894e-01,  6.3590e-03, -9.8479e-02,  5.5644e-02,  6.3759e-02,\n",
       "        -2.4756e-01,  1.6991e-01, -1.5741e-01, -6.2391e-02, -1.5357e-01,\n",
       "         2.6700e-01, -5.4306e-02,  2.2167e-01, -2.4215e-01,  3.6870e-02,\n",
       "        -2.0250e-02, -1.6752e+00,  4.5439e-01, -1.4031e-01,  9.6143e-02,\n",
       "         2.5538e-01, -7.5079e-02, -2.4472e-01,  1.1506e-01, -3.3222e-03,\n",
       "         1.6042e-01, -4.6712e-01,  1.2963e-01,  6.0436e-02, -1.8400e-01,\n",
       "         2.8130e-01, -1.4130e-01, -2.2920e-01,  2.5663e-02,  8.1771e-02,\n",
       "        -5.3451e-02,  7.5590e-02,  2.1095e-01, -2.2454e-01, -3.5566e-01,\n",
       "         9.7042e-02,  3.4690e-02, -4.0139e-02,  1.0555e-01,  1.5268e-01,\n",
       "         2.3273e-01,  3.1676e-03, -3.8688e-01, -1.2103e-02,  3.4583e-01,\n",
       "        -1.7757e-01,  1.4620e-01,  3.2625e-02, -6.6961e-02,  2.7557e-01,\n",
       "        -1.2879e-01,  2.1825e-01, -2.9028e-01,  2.3631e-01,  1.0715e-01,\n",
       "         8.0317e-02,  9.3672e-02,  3.4424e-01, -1.9515e-01,  4.9939e-01,\n",
       "         1.2911e-01,  2.8301e-01, -1.8161e-01, -6.5297e-01, -3.5723e-01,\n",
       "         2.7482e-01, -9.7540e-02, -7.8294e-02,  3.9118e-03,  8.7049e-02,\n",
       "         9.8073e-02,  1.7187e-01, -1.3128e+00,  4.0341e-02, -2.1126e-01,\n",
       "        -1.4133e+00,  3.4497e-02,  2.2168e-01, -1.1972e-01, -1.8170e-01,\n",
       "         7.6507e-02,  3.5284e-01,  1.7934e-01,  2.2132e-01,  1.5589e-01,\n",
       "        -2.6694e-02,  1.0254e-01,  2.1237e-01,  3.9665e-02,  2.5832e-01,\n",
       "        -6.5490e-02,  8.1651e-02,  4.2293e-01,  1.1638e-01, -1.0165e-01,\n",
       "        -1.4299e+00, -7.5346e-02,  2.0762e-01,  5.9546e-03,  1.7607e-01,\n",
       "         1.3833e-01, -8.1890e-02,  2.9936e-01,  1.3076e-01, -1.9170e-02,\n",
       "         4.6823e-02,  4.1869e-02, -3.0080e-02,  3.1885e-02, -2.7641e-01,\n",
       "         5.0565e-01,  9.2336e-01, -4.8878e-03, -2.4042e-01, -2.9086e-01,\n",
       "         3.3610e-02, -3.8636e-02, -1.2002e-01, -1.2162e-01,  1.4416e-01,\n",
       "        -2.9171e-01,  1.8036e-01, -2.5362e-01, -1.3048e+00, -1.6348e+00,\n",
       "         8.1851e-01, -2.3102e-01, -3.1643e-01,  2.7777e-01,  1.6438e+00,\n",
       "        -2.3653e-01,  2.9212e-01, -4.2063e-02, -7.4717e-02, -6.6824e-02,\n",
       "         3.0194e+00,  4.8920e-01, -1.7553e-01,  1.3955e+00,  1.9220e-01,\n",
       "         4.4193e-01, -4.3252e-01, -1.1715e+00, -2.0714e+00,  1.0374e-01,\n",
       "        -2.2206e-01, -8.4943e-02,  1.7619e-01, -1.3966e-01, -1.2033e-01,\n",
       "         8.3037e-02, -2.1788e-01,  7.7387e-02,  5.9542e-03, -3.6577e-01,\n",
       "        -3.1738e-01,  1.7339e-03, -3.6331e-01,  8.3944e-02,  4.9604e-01,\n",
       "         5.5472e-01,  9.1856e-03,  6.1593e-02, -7.9087e-02, -7.4032e-02,\n",
       "        -1.8425e+00, -1.2745e-01, -8.1035e-02,  2.0150e-01, -5.2407e-01,\n",
       "         4.0489e-01, -6.5969e-02,  3.8947e-01, -8.5902e-02,  4.4178e-01,\n",
       "        -7.9435e-02, -1.4692e-01,  6.0588e-02, -4.1873e-01, -1.6244e-01,\n",
       "        -2.4806e-02,  9.7696e-02, -3.2269e-01,  2.1695e+00,  3.2320e-02,\n",
       "        -5.2851e-01,  3.4637e-02,  3.4679e-01,  3.1954e-01, -4.7491e-01,\n",
       "        -1.6652e-02, -1.5067e-01,  3.3977e-01,  1.8096e-01, -1.1223e-01,\n",
       "        -1.2231e-01, -5.5304e-01,  2.3930e-01,  1.9227e-01,  5.6625e-02,\n",
       "         3.7857e-01, -4.5911e-02,  2.1671e-01, -7.0923e-01,  2.2015e-01,\n",
       "         1.4690e-01, -7.5900e-02, -5.1373e-01, -7.1579e-02,  1.4286e-01,\n",
       "         4.5658e-01,  2.2235e-01,  1.9589e-01, -3.3024e-02, -2.2519e-01,\n",
       "        -1.5834e-01, -3.0530e-01, -1.7149e-01, -1.1444e-01, -1.3002e-01,\n",
       "         1.9941e-01, -3.1554e-01, -3.2887e-01,  1.3655e+00,  3.0887e-02,\n",
       "         3.4886e-01,  9.9795e-01, -1.9126e-01, -2.5848e-01,  1.2505e-01,\n",
       "         2.5352e-01,  1.0091e-01, -3.2216e-02,  3.7801e-02, -4.9595e-03,\n",
       "         3.7634e-02,  3.0623e-01, -4.6119e-03,  4.9444e-02,  7.0467e-02,\n",
       "         1.8354e-01,  4.9053e-01, -3.9927e-02,  1.5473e-01, -2.0449e-01,\n",
       "        -5.6310e-01,  2.0023e-01, -6.7647e-03,  8.0406e-02,  3.3241e-01,\n",
       "        -1.9569e-02,  3.2762e-01, -1.0818e-02, -3.9559e-02,  2.6746e-02,\n",
       "        -5.3383e-02, -4.9223e-02,  8.8069e-02,  7.4499e-02, -1.2862e-01,\n",
       "         8.3818e-02,  8.9044e-02, -3.2918e-01, -3.9591e-02, -6.0169e-02,\n",
       "         1.5217e-01, -3.7972e-01,  2.9259e-01, -3.2933e-02, -3.0959e-01,\n",
       "         1.0105e-02, -9.3564e-02,  1.8286e-01, -2.2038e-02, -3.8897e-01,\n",
       "         1.2796e-01, -4.2997e-02, -4.5993e-01, -4.5395e-01,  1.4143e-01,\n",
       "        -1.0125e-01, -3.3328e-01, -2.0370e-01, -1.0622e-01, -1.4446e-01,\n",
       "         3.9442e-01,  1.6203e-01,  3.4184e-01, -9.8918e-02, -1.8157e-01,\n",
       "         6.2134e-01,  9.0894e-02,  9.2409e-02,  1.7832e-01, -2.1015e-01,\n",
       "         3.5673e-03, -2.1785e-01,  6.9074e-02, -5.5662e-01,  2.9039e-01,\n",
       "         1.7759e-01, -5.5246e-02, -2.1547e-01, -7.3353e-02,  2.6076e-02,\n",
       "        -1.6653e+00, -2.6401e-01, -2.0462e-01,  4.2286e-01,  1.7719e-01,\n",
       "        -3.1143e-01,  9.2326e-01, -1.6864e-01,  2.5520e-01, -2.3209e-01,\n",
       "        -2.2890e-01, -1.3660e-01,  4.9788e-02,  1.7430e-01, -5.5013e-01,\n",
       "         3.8338e-02, -1.1963e-01,  3.5242e-02,  5.9870e-02, -1.5274e-01,\n",
       "        -1.2710e+00, -2.9875e-01, -2.6766e-02,  3.7854e-02, -4.1871e-01,\n",
       "         7.2939e-03,  3.9711e-01,  5.8168e-02, -2.6348e-01,  3.8940e-01,\n",
       "         6.1589e-02, -2.4159e-01, -1.3307e-01, -2.7579e-01,  8.8090e-02,\n",
       "         5.8946e-01, -9.4648e-02, -4.0186e-02,  1.1377e-01,  8.2496e-02,\n",
       "        -1.3937e-01,  9.7846e-02, -2.5702e-01, -1.5037e-01,  4.8707e-01,\n",
       "         4.2791e-01,  6.4475e-02, -2.6957e-01, -2.5012e-01, -1.8231e-01,\n",
       "        -1.7173e+00,  1.1353e-01, -6.8776e-02,  3.7338e-01,  1.2811e-01,\n",
       "         2.6654e-01, -1.7055e-01,  2.4213e-02,  1.4748e-01,  9.6010e-02,\n",
       "        -3.8261e-01, -1.2902e-01, -2.1331e-01, -7.6057e-02,  2.4482e-01,\n",
       "         1.1073e-01,  4.5261e-01,  3.4344e-01, -2.1912e-01, -2.7792e-01,\n",
       "        -1.2535e-02,  5.0965e-02,  2.1709e-01,  7.9927e-03, -1.3441e-01,\n",
       "         2.5988e-01,  3.2231e-03,  1.7950e-01,  7.4354e-02, -3.2964e-01,\n",
       "         3.6002e-01, -8.7950e-01,  2.2727e-02, -9.6364e-02, -1.0990e-01,\n",
       "         1.6240e-01, -1.9400e-01,  4.3743e-01, -1.9510e-01,  4.6733e-01,\n",
       "         6.2145e-02,  5.2978e-01,  1.5302e+00,  2.8117e-01,  2.1851e-01,\n",
       "        -8.7874e-02, -5.9729e-02, -4.4249e-02,  5.7602e-01, -4.5091e-01,\n",
       "         1.7271e+00,  3.3866e-02, -1.5636e-01,  5.5481e-02,  8.1072e-01,\n",
       "         2.6444e-01,  1.8555e+00, -1.9656e-01, -6.4780e-02,  1.1188e-01,\n",
       "         8.7733e-02,  1.0594e-01, -1.9465e-01,  5.5264e-03, -1.2092e-01,\n",
       "         1.2105e-01, -9.2932e-01, -2.5885e-01,  3.4338e-02,  8.0015e-01,\n",
       "         1.4818e-03, -3.3949e-01,  1.6023e-01, -1.0392e-01, -2.0958e-02,\n",
       "         3.2421e-02, -3.1569e-01, -1.4836e-01, -4.0222e-02,  1.4063e-01,\n",
       "         3.1300e-01, -1.9539e-01, -9.6932e-02, -4.1362e-02, -2.0011e-01,\n",
       "        -8.8363e-02, -1.4534e-01,  4.4918e-01, -2.0424e-01,  2.3753e-01,\n",
       "         4.5989e-01, -1.8435e-01,  4.5101e-02,  2.6835e-01, -7.1969e-02,\n",
       "        -4.8811e-01, -3.6325e-01, -1.8532e-01,  3.1506e-02,  2.1642e-01,\n",
       "         1.9113e-01, -4.5536e-02, -2.5627e-01], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)\n",
    "model = BertModel.from_pretrained('/dev/bert/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor([101, 108]).unsqueeze(0)  # Batch size 1\n",
    "outputs = model(input_ids)\n",
    "last_hidden_states = outputs[0][0][0]  # The last hidden-state is the first element of the output tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor(tokenizer.encode(\"[CLS] My sentence\")).unsqueeze(0)\n",
    "outputs = model(input_ids)\n",
    "embeddings_of_last_layer = outputs[0]\n",
    "cls_embeddings = embeddings_of_last_layer[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.3212, -0.0544, -0.0861,  ...,  0.7931, -0.3903, -0.0120],\n",
       "         [ 0.3208, -0.0570, -0.0851,  ...,  0.7936, -0.3919, -0.0130],\n",
       "         [ 0.3468, -0.2333, -0.0710,  ...,  0.6953, -0.6291, -0.1370],\n",
       "         [ 0.2627, -0.3894, -0.3073,  ...,  0.9721, -0.4724, -0.2921],\n",
       "         [ 0.2843, -0.2475,  0.1256,  ...,  0.7671, -0.4177, -0.0446]]],\n",
       "       grad_fn=<NativeLayerNormBackward>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings_of_last_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 3.2121e-01, -5.4417e-02, -8.6102e-02,  1.6083e-01, -1.2724e-01,\n",
       "        -1.1863e-02, -3.9340e-01,  2.0404e-01,  1.0924e-01, -2.6062e-01,\n",
       "         5.3292e-01, -1.6562e-01,  6.3398e-02, -1.1186e-01, -4.1014e-01,\n",
       "         1.0758e-02, -5.5582e-02,  7.6777e-01,  4.5858e-01,  6.6396e-01,\n",
       "         8.7407e-02,  3.2365e-01, -3.0692e-01,  9.4165e-03,  4.5095e-01,\n",
       "        -2.1449e-01,  8.1025e-02, -2.0580e-01,  1.9197e-01,  7.4378e-02,\n",
       "         5.2771e-01,  3.0615e-01, -1.5516e-01,  5.7969e-01,  3.9827e-01,\n",
       "         3.7909e-02, -8.0676e-01, -1.9927e-01, -3.1221e-01, -4.4068e-01,\n",
       "        -3.3501e-01, -1.3054e-01,  3.9339e-01,  2.7643e-01,  4.4010e-01,\n",
       "         7.1446e-01,  2.0268e-01, -9.1150e-02,  4.2229e-01, -8.6419e-01,\n",
       "         5.4797e-02, -7.6668e-01,  7.8603e-01, -1.3261e+00,  4.0646e-01,\n",
       "         4.3429e-01,  2.8713e-03, -1.3567e-01, -1.9692e-01,  2.4281e-01,\n",
       "         4.0501e-01,  7.5429e-01,  5.1152e-02, -3.6600e-01, -5.5143e-01,\n",
       "        -5.5313e-01, -7.7702e-02,  1.8873e-01, -2.6089e-01, -2.8797e-01,\n",
       "         3.7144e-01,  1.5661e-01,  7.7181e-02,  3.7555e-01,  2.8242e-03,\n",
       "        -3.7789e-01,  1.5928e-01, -1.5426e-02, -5.2080e-01,  2.9001e-01,\n",
       "        -4.7022e-02,  5.0310e-01, -5.7958e-01,  1.8806e-01,  7.9406e-02,\n",
       "         3.6172e-01,  3.0478e-01, -5.8954e-02, -5.0865e-01, -4.8668e-01,\n",
       "        -3.3009e-01, -5.9086e-02, -5.9078e-02,  6.4257e-01, -3.3138e-01,\n",
       "         1.6775e-01,  4.1515e-01, -7.6415e-02, -1.0550e-01,  1.9442e-02,\n",
       "         4.1737e-01,  1.1019e+00,  4.6878e-01, -3.3798e-01, -1.1623e+00,\n",
       "        -9.5222e-02, -7.0271e-02, -3.2087e-01,  5.5232e-01,  1.7343e-01,\n",
       "         4.6734e-01, -1.8842e-01,  4.2280e-01, -2.6187e-01, -8.5913e-01,\n",
       "        -8.9236e-02, -6.5026e-01, -2.5124e-01, -2.8913e-01, -1.9076e-01,\n",
       "         5.3347e-01, -2.5563e-01,  1.2582e-01, -2.6777e-01,  3.2328e-01,\n",
       "        -5.7565e-01,  4.8893e-02, -4.6931e-01,  2.4059e-01,  4.1125e-01,\n",
       "         1.6142e-01,  1.2615e+00,  2.5668e-01, -2.2737e-01, -8.9056e-02,\n",
       "         6.4458e-01,  1.5852e-01,  3.5180e-01,  3.7759e-01,  2.4057e-01,\n",
       "         1.0233e-01, -2.8202e-01,  4.6634e-02, -1.3022e-01, -2.9929e-01,\n",
       "         1.4629e-01,  1.5215e-01,  3.3856e-01, -6.7959e-02, -4.6274e-01,\n",
       "        -3.7122e-01, -6.1834e-01,  2.9063e-02, -1.0869e-02,  9.0912e-02,\n",
       "        -1.9711e-01, -1.8797e-01, -4.8493e-01, -1.9138e-03,  6.9900e-01,\n",
       "         1.0613e+00,  5.1097e-02, -2.1475e-02,  1.0091e+00, -2.0686e-01,\n",
       "        -1.3473e-02, -1.6269e-01, -1.1405e-01,  8.4124e-02, -4.9399e-01,\n",
       "        -5.3900e-01, -1.5578e-01,  4.1504e-01, -6.8432e-01, -4.1785e-01,\n",
       "        -8.7895e-02,  1.5628e-01,  7.0994e-02, -9.6025e-02, -8.2851e-02,\n",
       "         1.0557e-02,  1.2285e-02,  1.8125e-01,  6.7220e-02,  3.7552e-01,\n",
       "         4.5214e-01, -2.2771e-01, -1.7922e-01, -2.2132e-01, -2.2351e-01,\n",
       "         4.1567e-01,  4.5525e-01,  3.2668e-01, -3.5555e-01,  4.5705e-01,\n",
       "         8.1493e-01,  5.2048e-01,  5.8965e-01, -2.3094e-01,  2.6996e-01,\n",
       "         4.9183e-02,  1.7530e-01, -7.5326e-02,  2.8280e-02, -5.9977e-01,\n",
       "         1.1981e+00,  2.6628e-01, -6.0993e-01, -1.8561e-01,  3.9010e-01,\n",
       "        -2.8663e-01, -2.8219e-02,  2.4922e-01,  8.6136e-02,  1.4146e-01,\n",
       "         7.9266e-02, -4.4460e-01,  3.4256e-02,  1.2146e-01, -1.2156e+00,\n",
       "        -1.5603e-01, -1.0983e-02, -2.6043e-02,  1.8989e-01, -3.4790e-01,\n",
       "        -4.7284e-01, -1.5810e-01,  9.7915e-02,  6.4172e-01, -1.5573e-01,\n",
       "         7.5969e-01, -1.2794e-01,  4.4235e-01, -8.3166e-01,  2.4276e-02,\n",
       "        -3.4779e-01,  7.3097e-01, -3.5733e-01, -5.3923e-01, -3.3503e-01,\n",
       "        -1.1764e-01,  1.4630e-02,  1.8480e-01, -1.1747e-01,  4.1928e-01,\n",
       "         1.5273e-01,  7.9482e-01, -4.4439e-02,  9.4490e-03, -3.2917e-01,\n",
       "         3.2970e-01, -6.1592e-02,  6.2582e-01, -2.5595e-01, -1.1037e-01,\n",
       "         3.5643e-01, -1.5477e-01, -3.0433e-01,  4.0322e-01, -2.1236e-02,\n",
       "        -3.7792e-01, -7.6729e-01, -2.9571e-01, -5.6294e-02, -4.1492e-01,\n",
       "        -3.4718e-02, -1.6132e-02, -1.3161e-01,  1.3560e-01,  4.2329e-01,\n",
       "        -8.6717e-02, -8.3248e-03,  3.9778e-02, -1.2462e-01,  9.4678e-02,\n",
       "        -2.4480e-03, -8.0626e-02,  5.2786e-01, -5.9247e-01,  4.7246e-02,\n",
       "        -1.3518e-01, -7.4530e-03, -3.4991e-01, -1.2097e-01,  3.8708e-01,\n",
       "         1.2516e-01,  2.7170e-01, -8.1706e-02, -1.5936e-01, -9.5434e-01,\n",
       "        -1.6797e-01, -1.5231e-01, -5.3536e-02,  9.7531e-02,  1.2862e-01,\n",
       "        -1.1209e-01, -4.9015e-01,  1.9368e-01, -4.1125e-01,  2.3129e-01,\n",
       "         1.8500e-01, -6.8491e-01, -1.3518e-01, -7.9098e-02, -1.1357e+00,\n",
       "         3.5645e-01, -3.8189e-01, -1.1066e-01,  4.1466e-01,  1.8693e-01,\n",
       "        -1.1517e+00,  1.7977e-01,  3.1681e-01, -3.6491e-01, -3.8542e-01,\n",
       "        -2.4248e-01, -9.1871e-02,  1.8148e-01, -5.4503e-02, -9.0407e-02,\n",
       "        -2.6425e-02, -1.1512e+00, -1.6812e-01,  4.7055e-01,  2.6242e-01,\n",
       "         6.1527e-01, -4.1731e-01,  1.2540e-02, -1.5770e-01, -2.3876e-01,\n",
       "         2.5393e-01,  1.3791e+00, -1.1134e-01, -4.5206e-01, -4.6975e-01,\n",
       "         3.2057e-02, -3.6115e-01, -4.6197e-01, -3.0084e-01, -7.6848e-02,\n",
       "        -7.0268e-02, -2.6445e-01,  4.1583e-01, -7.5797e-02,  1.6946e-01,\n",
       "         4.8545e-01,  1.1429e-01, -5.6821e-01,  1.2276e-01,  5.6066e-01,\n",
       "         5.8366e-01,  1.4752e-01, -2.8527e-01,  2.2361e-01,  1.6545e-01,\n",
       "         3.0824e-01, -1.2940e-01, -7.0895e-02, -3.1877e-01,  2.3901e-01,\n",
       "        -2.8364e-01,  2.1514e-01, -1.7823e-01,  1.9859e-01, -1.2770e-01,\n",
       "         3.1656e-01,  8.0198e-02, -4.7395e-01,  4.0608e-01, -1.7106e-01,\n",
       "         7.7819e-01,  6.4274e-01, -6.7364e-01, -9.7298e-01,  4.3283e-02,\n",
       "         2.7069e-02, -5.7917e-01,  2.3391e-01,  8.9862e-02,  3.0602e-01,\n",
       "         6.4259e-01,  1.8256e-01, -7.5342e-01,  4.5945e-01,  3.4500e-02,\n",
       "        -7.5729e-01,  4.6952e-01,  2.4955e-01,  2.4289e-02, -1.9983e-01,\n",
       "        -9.6951e-02, -2.1889e-01,  4.7290e-01, -7.0018e-02,  2.5906e-01,\n",
       "        -3.9812e-01,  1.1195e-01, -2.4295e-01,  1.9157e-01,  1.2948e-01,\n",
       "        -1.6094e-01, -1.2455e-01, -2.6483e-01,  1.5492e-01, -1.6052e-01,\n",
       "        -8.8797e-01,  5.6545e-02,  1.7351e-01, -2.8420e-01, -1.7831e-01,\n",
       "         1.1797e-01, -1.1247e-02,  2.5795e-01,  2.0759e-02,  1.5254e-01,\n",
       "         2.8556e-01, -1.7923e-01, -9.7890e-02,  3.4001e-01, -1.6098e-01,\n",
       "         2.7449e-01,  2.6229e-02,  1.3958e-01, -7.8790e-03, -7.6216e-02,\n",
       "         1.3675e-01,  3.1057e-01, -1.9611e-01,  7.3224e-01,  7.9114e-02,\n",
       "        -1.0887e-01,  3.2770e-01,  4.6165e-01, -5.3929e-01, -1.0418e+00,\n",
       "         2.8454e-01, -1.5729e-01,  3.9128e-01,  3.7318e-03,  1.0719e+00,\n",
       "         2.4417e-01,  5.6403e-01, -2.2635e-01,  1.9795e-02, -4.8456e-01,\n",
       "         2.0106e+00,  3.9742e-01,  6.7391e-02,  6.7550e-01,  2.0472e-01,\n",
       "         6.2898e-02, -6.5656e-01, -1.0729e+00, -8.2952e-01,  2.7921e-02,\n",
       "        -1.0521e-01, -3.3537e-01,  3.1785e-01, -6.5229e-02,  3.0698e-01,\n",
       "        -8.5835e-02,  2.0394e-01, -3.0661e-01,  6.8070e-02, -2.1957e-01,\n",
       "        -2.6821e-01, -5.4792e-01,  5.7147e-02, -8.0511e-02,  4.1313e-02,\n",
       "         4.5473e-01,  5.3391e-02, -2.0403e-01,  1.5874e-02, -3.8150e-01,\n",
       "        -7.5306e-01,  5.4094e-01,  1.6498e-02, -5.0370e-01, -2.7454e-01,\n",
       "         3.6754e-01, -8.1986e-02,  5.7731e-02, -3.8272e-01, -3.6408e-01,\n",
       "         2.4023e-01, -3.1316e-01,  9.3045e-02,  1.2109e-01,  1.3321e-01,\n",
       "         6.2910e-01,  4.6686e-01, -7.1537e-02,  1.2150e+00, -7.4760e-01,\n",
       "         9.4265e-02,  4.8522e-01,  4.2003e-02,  1.2223e-01,  3.5880e-01,\n",
       "        -9.5306e-03,  7.6916e-02, -3.6211e-01,  5.3604e-01, -2.2438e-01,\n",
       "        -7.9574e-02, -1.6831e-01,  2.7119e-01, -1.3352e-01, -1.2763e-01,\n",
       "         3.1196e-01,  3.9533e-02, -1.2031e-01, -5.1775e-01,  1.5010e-01,\n",
       "        -5.2204e-01, -1.6894e-01,  2.2564e-01, -1.0504e-01, -2.2719e-01,\n",
       "         8.3528e-02,  5.6959e-01,  6.7296e-02, -3.2759e-01, -3.3009e-01,\n",
       "         1.9262e-01, -7.3367e-01, -5.0580e-01, -6.9450e-02,  1.0902e-01,\n",
       "        -1.2309e-01, -4.2111e-01, -3.2068e-01,  4.2559e-01,  7.4158e-02,\n",
       "         3.5367e-01,  5.4735e-01, -2.5104e-01, -7.4963e-02, -3.4801e-01,\n",
       "         4.2946e-01,  2.8449e-01, -1.7761e-01,  1.5135e-01, -3.4726e-01,\n",
       "         6.1502e-01,  6.1580e-03, -2.9058e-01, -4.5541e-02, -4.0483e-01,\n",
       "         8.7577e-02,  3.0776e-01,  2.0144e-01,  1.7191e-01, -1.0683e-01,\n",
       "         2.6898e-01,  1.1065e-01,  2.7686e-01, -5.0540e-01,  4.4701e-01,\n",
       "         2.4901e-01,  3.7757e-01, -2.3078e-01, -2.7129e-01,  2.9713e-01,\n",
       "        -3.4528e-01, -2.2666e-01,  5.9913e-01,  2.7307e-01,  3.5630e-02,\n",
       "        -1.3299e-01,  3.2675e-02,  2.1386e-01,  5.1458e-02,  2.0335e-02,\n",
       "        -1.0687e-01, -2.4467e-01, -2.1598e-01,  6.8944e-01, -1.0072e-01,\n",
       "         3.4363e-01, -4.8733e-01, -1.4908e-01,  7.8340e-02, -3.0102e-01,\n",
       "        -2.0687e-01, -8.7138e-02,  5.2777e-01, -4.4270e-01, -4.1237e-01,\n",
       "         1.3107e-01,  3.7468e-02,  5.3215e-01, -4.5455e-02,  5.1548e-01,\n",
       "         2.0458e-01,  6.1201e-01, -1.7302e-01, -5.7576e-02, -3.0444e-01,\n",
       "         9.7887e-02, -1.7164e-01,  7.4160e-02,  3.9998e-02, -4.9847e-01,\n",
       "        -1.8351e-02,  3.7422e-01,  4.5983e-01, -8.9537e-01,  1.6140e-01,\n",
       "        -4.1459e-01,  6.1069e-01, -2.7672e-01,  5.7716e-01,  4.3191e-01,\n",
       "        -1.8666e+00, -3.1564e-01, -1.0847e-01, -1.2668e-02,  3.6888e-02,\n",
       "         2.0462e-01,  4.4656e-01, -1.6199e-01, -5.3103e-01, -1.9796e-01,\n",
       "         2.5852e-01, -2.4495e-03, -1.1702e-01,  1.5607e-01, -5.6962e-01,\n",
       "        -3.8247e-01, -3.1790e-01,  1.8849e-02, -2.6666e-01, -2.0565e-01,\n",
       "        -4.5519e-01, -2.3184e-01, -6.4309e-01,  5.8629e-02,  1.4029e-02,\n",
       "         3.7407e-01,  4.0038e-01, -1.0436e-01, -1.6506e-01,  1.7766e-01,\n",
       "        -4.0476e-01, -6.5355e-01,  1.2543e-01, -3.7476e-01,  4.7781e-01,\n",
       "         3.6970e-01, -4.2640e-02, -2.1734e-02, -2.9834e-01, -2.7091e-01,\n",
       "         2.6339e-01, -1.8341e-01, -1.5682e-01, -1.7431e-02,  1.4702e-01,\n",
       "         1.9701e-01,  1.2136e-01, -6.3585e-03,  1.9522e-01,  1.1006e-01,\n",
       "        -6.9593e-01,  1.3422e-02, -6.9740e-02,  1.8673e-01, -3.7510e-01,\n",
       "         2.7030e-01,  2.0478e-01,  2.1123e-01,  1.6025e-01,  1.3948e-01,\n",
       "        -2.8445e-01, -1.8620e-01,  5.4231e-01,  5.8034e-02,  9.8049e-02,\n",
       "        -4.5030e-01,  1.2212e-01,  2.6569e-01, -1.3040e-01, -3.8002e-02,\n",
       "        -1.5654e-01, -1.5628e-01,  1.5931e-01, -6.7959e-02,  3.8645e-01,\n",
       "         1.6365e-01,  3.3953e-02,  1.3182e-01, -1.1300e-01, -2.9650e-01,\n",
       "        -1.2673e-01, -5.3324e-01,  2.8687e-01,  2.0191e-01, -2.8502e-01,\n",
       "         3.8137e-03, -7.8539e-02,  4.3298e-02,  2.3019e-01,  5.3615e-02,\n",
       "         1.1291e-01,  6.7381e-01,  8.3750e-01,  1.1227e-01,  2.4424e-02,\n",
       "         5.1538e-02,  2.2896e-01,  2.0317e-01,  2.8887e-01, -4.6197e-01,\n",
       "         7.5221e-01, -1.7739e-01,  1.1611e-01,  7.8691e-03,  8.5874e-01,\n",
       "         3.7989e-01,  1.4353e+00,  7.2204e-02, -2.7456e-01,  2.9294e-01,\n",
       "        -1.2900e-01, -1.6645e-01,  1.8344e-01, -2.7216e-02,  3.1686e-01,\n",
       "        -9.1761e-02, -1.7467e+00, -1.6763e-01,  1.2551e-01, -5.6322e-01,\n",
       "         2.5933e-01, -4.1665e-01,  2.1233e-02,  4.1431e-01, -3.6543e-01,\n",
       "        -5.0737e-02, -1.2092e-01, -3.5924e-01,  1.7864e-01, -3.1055e-02,\n",
       "        -1.7180e-01, -1.3339e-01,  1.5168e-01, -4.6358e-01, -6.5227e-01,\n",
       "         1.3538e-01, -8.7437e-01,  1.6259e-03, -3.7329e-01,  1.1586e-01,\n",
       "         4.7505e-01,  3.9209e-01, -4.3146e-01,  3.7676e-01, -8.6242e-02,\n",
       "        -2.9900e-01, -1.2615e-02,  1.9313e-02,  4.3178e-01,  3.3255e-01,\n",
       "         7.9307e-01, -3.9033e-01, -1.2022e-02], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_embeddings[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 79])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([79, 768])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2.6847e-01, -5.7668e-01,  1.8359e-01,  1.1416e-01,  8.5621e-02,\n",
       "         1.0435e-01,  2.6839e-02,  9.4382e-01, -5.5169e-01,  3.6284e-01,\n",
       "         1.9132e-01,  7.8991e-01,  2.5428e-01,  4.9141e-01,  2.7798e-01,\n",
       "        -2.1794e-01,  8.7347e-01, -1.7157e-01, -6.0503e-01,  7.7051e-01,\n",
       "        -7.8044e-01, -6.4269e-02,  8.8347e-01, -1.6645e-01,  1.4419e-01,\n",
       "         1.1550e+00, -7.6593e-01, -6.0788e-01,  3.1496e-01, -3.2022e-01,\n",
       "        -5.0232e-01, -5.0274e-01,  3.3638e-03,  3.2646e-01,  1.1946e-01,\n",
       "        -8.0921e-02,  1.6555e-02,  5.6689e-01,  6.9221e-01,  8.3903e-02,\n",
       "         1.0178e-03,  2.4890e-01, -8.7362e-02, -3.2178e-01, -8.0344e-02,\n",
       "        -3.2293e-01,  3.9753e-01,  2.2282e-01, -3.5801e-01,  9.8585e-01,\n",
       "         5.6671e-02, -1.5473e-01, -1.7192e-01,  1.1102e-01, -9.4336e-01,\n",
       "        -1.8301e-02, -2.4016e-01,  4.5593e-01,  6.9304e-01,  3.2511e-01,\n",
       "        -2.6158e-01, -2.1500e-02, -1.4318e-01, -8.7951e-02, -5.5176e-01,\n",
       "        -1.1372e-01, -1.9628e-01,  2.6121e-01,  1.2038e-02, -3.1737e-01,\n",
       "        -6.3956e-01,  5.6175e-01,  3.2111e-01,  1.0486e-01, -5.8110e-01,\n",
       "         1.0422e+00,  2.1251e-01,  1.7870e-01, -8.5082e-01, -4.9337e-01,\n",
       "         1.3393e-01,  6.4494e-01,  3.2601e-01,  5.7332e-01,  3.1158e-01,\n",
       "         3.0407e-01,  9.4088e-01, -3.9975e-01,  6.5441e-01, -6.9658e-01,\n",
       "         4.1638e-01,  4.8662e-01,  1.4325e+00,  4.4359e-01,  1.1182e-02,\n",
       "         9.0761e-01,  3.0732e-01, -4.7943e-02, -1.8414e-01,  3.9811e-01,\n",
       "        -9.7828e-01, -3.1439e-01,  1.1945e+00, -1.4564e-01,  9.1123e-01,\n",
       "        -2.3278e-01,  1.9222e-01, -3.0016e-01, -4.3851e-02,  1.0259e+00,\n",
       "         1.9957e-01,  8.2302e-01,  3.8845e-02,  3.7259e-01,  2.3385e-01,\n",
       "         3.3179e-02,  9.6359e-02, -2.8075e-01,  2.6068e-01, -6.6820e-02,\n",
       "        -3.0295e-01, -3.5409e-01, -3.6058e-01,  4.9243e-01, -6.4668e-01,\n",
       "        -2.0351e-02, -3.9858e-02, -2.6134e-01,  7.6119e-01, -5.7500e-02,\n",
       "        -5.2630e-01, -2.5723e-01, -3.8605e-01,  1.2910e-01, -2.5467e-01,\n",
       "        -5.3825e-01,  3.4559e-01,  1.5016e-01, -5.1615e-01,  6.8652e-01,\n",
       "         2.5321e-01,  5.7259e-03, -7.6572e-01,  1.8465e-02, -1.6997e-01,\n",
       "         1.0909e-01, -6.7449e-01,  1.2521e-02,  4.2419e-02, -1.3194e-01,\n",
       "        -1.5870e-01,  4.7160e-01,  7.1617e-01,  5.8928e-01, -4.1310e-01,\n",
       "         4.4685e-01,  6.5288e-01, -7.5490e-02, -1.4777e-01, -1.2105e-01,\n",
       "        -4.8221e-01, -3.0670e-01, -3.8637e-01,  2.3884e-01, -1.8905e-01,\n",
       "        -4.3778e-01,  3.0675e-01, -3.7843e-01, -3.7980e-01,  3.6563e-01,\n",
       "         1.2942e-01,  4.9877e-01, -5.7114e-02,  3.2089e-01,  5.0205e-01,\n",
       "         6.2916e-01, -6.1401e-01, -5.6677e-01, -1.0266e-01, -4.5593e-01,\n",
       "        -4.6118e-01, -3.0126e-01,  5.2984e-01,  2.4797e-02,  6.4021e-02,\n",
       "        -3.0325e-01,  2.2951e-01, -9.4534e-02, -1.2687e-01,  2.1987e-02,\n",
       "         1.3973e-01,  3.2371e-02,  9.4441e-02, -1.7373e-01, -3.1845e-01,\n",
       "        -9.5568e-01, -1.1716e-01,  4.4229e-01,  8.3453e-02, -3.1996e-01,\n",
       "        -7.4940e-01, -3.8540e-02,  4.5093e-01,  6.1047e-01,  3.0276e-02,\n",
       "         7.3559e-01, -1.4809e-01, -2.4077e-01, -4.3576e-01,  1.2029e-01,\n",
       "         2.1741e-01,  6.1448e-03,  2.0090e-01,  7.3106e-02,  5.0192e-01,\n",
       "         8.7100e-02, -2.6740e-01,  3.3130e-01,  5.0205e-01,  8.2430e-01,\n",
       "        -3.2636e-01,  1.8863e-01, -8.4812e-02,  8.0577e-02, -1.4178e-01,\n",
       "        -1.2988e-01, -1.7501e-01,  3.1857e-01, -7.2378e-01, -3.3532e-02,\n",
       "        -3.5961e-01, -2.6669e-01, -1.4139e-01,  7.7836e-01, -2.7147e+00,\n",
       "         4.0693e-01, -3.3339e-01, -9.1075e-02, -3.5765e-01, -1.6179e-02,\n",
       "        -5.5474e-01,  6.5054e-01,  1.0654e-01,  1.1963e-01,  1.9362e-01,\n",
       "         9.2375e-02, -9.2573e-02,  3.0658e-01, -5.0338e-02,  4.0178e-01,\n",
       "         1.0539e-02, -1.4885e-01, -2.3678e-01, -3.7580e-01, -2.3087e-01,\n",
       "         2.1630e-01,  4.3854e-01,  2.5550e-02,  3.0947e-01, -2.3964e-01,\n",
       "         5.3726e-01, -1.1796e+00,  6.4010e-01,  2.7522e-01,  2.3310e-02,\n",
       "         2.1260e-01,  1.1499e-01, -1.0421e-01, -1.9796e-01, -1.3604e-01,\n",
       "        -3.5293e-01,  6.2500e-01,  5.2075e-01,  4.3842e-01, -5.4225e-04,\n",
       "         9.6605e-01, -7.0358e-03, -2.2508e-01,  1.3598e-01,  1.7643e-01,\n",
       "         5.3025e-01, -3.2814e-01, -9.3184e-03,  2.1229e-01,  1.2141e-01,\n",
       "         2.4531e-01, -7.1704e-02,  1.4056e-01,  3.9564e-01, -4.2203e-02,\n",
       "        -1.8448e-01, -6.1301e-01, -1.0951e-01,  3.7001e-01,  2.1024e-01,\n",
       "         1.8076e-01, -5.5628e-01,  7.1570e-01,  1.8564e-01, -2.6056e-01,\n",
       "        -1.0765e-01,  1.8831e-01, -4.3336e-01, -1.0590e-01, -2.9067e-01,\n",
       "        -4.8510e-01, -3.1054e-01, -2.5609e-03,  1.1966e+00, -1.1558e-01,\n",
       "        -2.2113e-01,  1.9979e-01, -1.0550e-01,  8.5308e-01,  9.5032e-02,\n",
       "        -4.8253e-01,  5.8335e-02, -4.3467e-01, -1.1050e+00,  4.8754e-01,\n",
       "        -7.5398e-02, -3.3567e-01,  3.6128e-01,  1.1553e+00,  2.5805e-01,\n",
       "         2.3063e-01,  1.9884e-01, -6.6995e-01,  6.1784e-01, -3.3605e-02,\n",
       "         6.8470e-01,  1.7035e+00, -6.1520e-01, -2.7629e-01,  1.6232e-01,\n",
       "         4.5481e-01, -1.0702e-01,  6.6487e-02, -3.6145e-01, -3.6565e-01,\n",
       "         2.9072e-01, -4.3532e-01,  5.2650e-01, -5.3672e-01, -2.7175e-01,\n",
       "        -2.0354e-02, -2.8176e-01,  8.0006e-01, -4.5230e-01, -9.4270e-01,\n",
       "        -1.2824e-01,  3.7585e-01,  5.6828e-01,  4.3386e-01, -2.1580e-01,\n",
       "         4.7182e-02, -7.8153e-01,  2.6612e-01, -2.7267e-02, -5.8148e-01,\n",
       "         4.9046e-01,  2.8335e-01, -2.2245e-01, -1.0174e-01, -5.3120e-01,\n",
       "        -5.9842e-02, -1.5781e-01, -4.0598e-01, -6.3554e-01,  6.8757e-01,\n",
       "         1.8569e-01,  9.0452e-01, -5.9145e-01, -8.3117e-01,  1.6769e-02,\n",
       "        -3.2172e-01,  1.7398e-01,  3.3302e-02,  1.4256e+00,  7.2871e-03,\n",
       "        -1.2942e-01, -2.4012e-02,  1.7846e-01,  1.0885e-01,  2.4285e-01,\n",
       "         3.3560e-01, -3.7233e-01,  1.4131e-01,  9.5552e-01, -7.0894e-01,\n",
       "         4.0000e-01,  1.8965e-01,  3.7539e-01,  8.1786e-02, -6.6953e-02,\n",
       "         5.3116e-02,  1.9716e-01,  5.2855e-02, -7.8140e-01,  3.1883e-01,\n",
       "         2.2974e-01,  5.3384e-01,  5.2965e-01,  2.3138e-01, -2.8012e-01,\n",
       "         7.4355e-01,  1.5680e-01,  1.6936e-01, -5.8205e-01, -5.9666e-01,\n",
       "         3.8869e-02,  3.3127e-01,  5.8263e-02,  5.8429e-01,  2.2894e-01,\n",
       "         1.0173e+00, -1.0290e+00, -7.2563e-02,  2.7593e-02, -1.4118e-01,\n",
       "         1.1833e-01,  1.7892e-01,  2.2087e-01, -1.5093e+00, -3.8624e-02,\n",
       "         9.8351e-02, -4.6229e-01, -1.6063e-01, -6.3503e-01, -7.7836e-01,\n",
       "        -5.9275e-01, -3.4319e-01, -8.1302e-01,  4.3185e-01,  1.2143e-01,\n",
       "        -1.5148e-01,  3.0685e-01,  1.7908e-01, -7.4402e-01,  1.0756e-01,\n",
       "         9.6134e-02, -2.6671e-02,  1.6272e-01, -1.8772e-01,  2.7444e-01,\n",
       "        -2.5031e-01, -4.0691e-02, -7.9517e-01, -2.1036e-01,  6.0171e-01,\n",
       "        -6.2841e-01,  2.0435e-02, -3.3704e-01, -2.0260e-01, -4.5876e-01,\n",
       "        -1.0905e-01, -6.3734e-01, -9.4440e-02, -2.7557e-03, -8.5352e-01,\n",
       "        -1.7694e-01,  4.6021e-01,  8.9888e-02, -3.2114e-02,  5.3926e-01,\n",
       "        -7.8756e-02, -5.4275e-01, -3.5428e-02,  2.4586e-01,  2.4856e-01,\n",
       "        -5.2556e-01,  6.2264e-01, -2.9780e-01, -7.6469e-01,  3.8452e-01,\n",
       "         3.7923e-02,  3.1273e-01, -6.9236e-01,  4.6148e-01, -6.5221e-01,\n",
       "         9.1551e-02,  2.4697e-01,  9.7381e-01,  4.3933e-02, -1.0014e-01,\n",
       "         6.3546e-01,  1.0109e-01,  3.9594e-01, -1.0641e-01,  3.0489e-01,\n",
       "         4.0822e-01,  7.4310e-02,  7.9089e-02, -1.3605e+00, -7.7940e-02,\n",
       "         4.5538e-02, -1.5621e+00, -4.0787e-01,  1.0745e+00,  1.0456e+00,\n",
       "        -1.7362e-01,  4.9207e-01, -3.9849e-01, -3.5433e-01, -2.9356e-01,\n",
       "         1.4150e-01, -6.5127e-02, -2.4184e-02, -3.6887e-01,  3.4649e-01,\n",
       "        -5.0568e-01, -1.0123e+00, -2.5136e-01,  3.4487e-01, -7.5039e-01,\n",
       "         1.8940e-01,  6.0301e-01,  5.4033e-02,  3.6487e-02,  3.4390e-01,\n",
       "        -3.1791e-01,  3.4212e-01, -2.3783e-01,  3.2811e-01, -2.3598e-01,\n",
       "        -1.4944e-01,  6.0043e-02, -1.6833e-01, -3.9272e-02, -3.8345e-02,\n",
       "        -1.3501e-01, -8.5075e-02, -1.8398e-01,  3.1225e-01, -7.3911e-05,\n",
       "         8.7433e-01,  9.1672e-02, -5.1937e-01, -1.8420e-01,  2.5702e-03,\n",
       "        -2.5434e-01,  3.3714e-01, -3.6582e-01,  4.0246e-01,  6.8649e-01,\n",
       "         9.4397e-02, -3.9191e-01, -1.5119e-01, -6.9656e-01,  1.3464e-01,\n",
       "         6.4089e-01, -2.2787e-01, -2.7062e-01, -1.9762e-03,  3.1456e-01,\n",
       "         3.4523e-01,  3.5623e-01, -6.2910e-01,  3.5497e-02,  1.4997e-01,\n",
       "        -4.9553e-01,  6.2061e-02, -3.9483e-02, -2.0136e-01,  4.6614e-02,\n",
       "        -1.5808e-01, -8.7976e-01, -3.8718e-01,  6.7605e-02,  3.7298e-01,\n",
       "        -3.1142e-02,  2.9532e-01, -1.4039e-01, -7.6977e-02, -9.1358e-02,\n",
       "         5.2847e-01,  5.5834e-01,  3.2359e-01, -5.5441e-01, -3.9225e-01,\n",
       "        -1.7943e-01,  7.2463e-02,  3.1222e-01,  5.7815e-03,  5.4748e-01,\n",
       "        -2.4257e-02, -5.2004e-01,  4.2463e-01,  2.4496e-01, -4.8322e-03,\n",
       "        -1.6223e-02,  1.7738e-01, -8.3908e-01,  3.2045e-01,  1.4782e-01,\n",
       "        -3.0989e-01,  2.5227e-01, -2.4796e-01, -5.0736e-02, -9.4472e-02,\n",
       "         3.3228e-01, -2.5357e-01, -5.7820e-03, -4.7399e-01, -4.7797e-01,\n",
       "        -1.6566e-01, -3.2607e-01,  2.5339e-01, -1.0371e+00, -8.2024e-01,\n",
       "        -3.1554e-01, -3.1020e-01, -2.9275e-01, -6.4210e-02, -1.1195e-01,\n",
       "        -1.3001e-01, -9.7234e-01,  8.2013e-01,  3.4441e-01,  4.9961e-01,\n",
       "        -9.5211e-01, -4.5365e-01,  5.7796e-02, -9.2079e-02,  4.2470e-01,\n",
       "         3.7397e-01,  3.0466e-01,  2.3136e-01,  5.8939e-01,  3.7875e-01,\n",
       "         1.1963e-01, -4.1086e-01, -1.2182e-01, -1.5108e-01, -2.1434e-02,\n",
       "         6.6627e-01, -6.1243e-01,  6.9204e-02,  5.7770e-02,  4.5716e-02,\n",
       "        -1.3229e-01, -2.9565e-01,  2.6573e-01,  2.3458e-01, -2.2313e-02,\n",
       "         3.0157e-01,  3.9161e-01,  5.4273e-01, -6.6594e-02, -2.4994e-01,\n",
       "         7.5517e-01, -4.1309e-01,  1.7800e-01, -4.5480e-01,  3.0094e-01,\n",
       "         7.2354e-01, -1.3868e-01,  4.2747e-01, -4.7491e-01,  2.5442e-01,\n",
       "        -4.0451e-01,  6.7476e-01, -4.0120e-01, -2.3793e-01,  2.5427e-01,\n",
       "         6.3564e-01, -6.4397e-01, -1.7057e-01,  2.8693e-02, -3.7835e-01,\n",
       "        -3.2159e-01, -3.5542e-01,  6.5670e-02,  7.8073e-01,  9.1985e-01,\n",
       "         3.4446e-01,  2.4414e-01,  6.6289e-02,  4.2853e-01,  2.4971e-01,\n",
       "         5.8070e-02,  2.5971e-01,  5.0954e-01, -1.5267e-02, -1.9897e-01,\n",
       "        -1.9712e-01, -1.7045e-01, -3.1585e-01, -3.2779e-01, -1.2330e-01,\n",
       "         4.4529e-01,  3.6209e-01,  2.1750e-01,  1.4006e-01, -1.5886e+00,\n",
       "        -8.3625e-02, -3.7250e-01,  2.4365e-01,  1.7099e-01,  6.4124e-01,\n",
       "         1.2644e-01, -6.2873e-01,  2.8909e-01, -4.5855e-01, -6.5290e-01,\n",
       "         6.3426e-01, -3.2720e-01, -3.5145e-01,  3.2973e-02, -2.2346e-01,\n",
       "         3.5733e-01, -2.6622e-01,  1.2988e-04, -2.9391e-01,  2.1321e-01,\n",
       "        -4.2438e-01,  3.2828e-01, -2.5595e-01, -3.6729e-01,  1.8632e-01,\n",
       "        -1.9012e-01, -3.2597e-01, -9.3324e-01,  5.2824e-01,  4.0791e-01,\n",
       "         5.1943e-02,  5.0802e-01, -6.1813e-01, -4.7995e-02, -4.2810e-01,\n",
       "        -3.2394e-02, -1.7879e+00,  1.8174e-01, -8.0677e-02,  6.4601e-01,\n",
       "        -5.4087e-01,  3.6845e-01,  6.8683e-01, -5.2261e-01,  2.1859e-01,\n",
       "         1.3869e-01, -3.3915e-01, -4.8060e-01, -2.3756e-01, -1.2214e-01,\n",
       "         2.7718e-01, -7.6832e-02,  1.0239e-01,  4.1892e-02, -3.8944e-01,\n",
       "         2.6836e-02,  5.7737e-01, -1.8214e-01,  2.5199e-01, -5.8666e-02,\n",
       "        -7.0572e-01, -1.1248e-02,  4.1526e-01, -6.2633e-03,  1.1287e+00,\n",
       "        -7.0173e-01, -8.6186e-01, -2.5107e-01,  1.7912e-02,  2.3472e-01,\n",
       "        -2.4831e-02,  1.4298e-02, -3.3218e-01], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = last_hidden_states.mean(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 768])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6.4805e-02, -4.2387e-01,  2.5238e-02, -4.5042e-02,  2.8164e-01,\n",
       "          1.2641e-01,  4.8142e-02,  4.2612e-01, -9.0083e-02,  2.7727e-01,\n",
       "          5.1189e-01,  1.4792e-02,  1.1726e-01,  3.6286e-01,  3.8785e-01,\n",
       "         -8.8801e-02,  4.5374e-01,  2.1557e-01, -3.9564e-01,  4.0138e-01,\n",
       "         -8.2246e-01,  8.9695e-02,  7.3050e-01, -3.4167e-01,  2.1496e-01,\n",
       "          6.6250e-01, -4.7407e-01, -3.1473e-01,  2.2099e-01, -2.9732e-01,\n",
       "         -4.0190e-01, -1.1781e-01, -1.4600e-01,  2.1357e-01, -1.3338e-02,\n",
       "         -1.6685e-01,  1.7888e-01,  6.5907e-02,  3.6534e-02,  6.4048e-02,\n",
       "          3.4789e-01,  2.4846e-01,  1.1392e-01, -1.7823e-01, -2.4612e-01,\n",
       "         -1.1902e-01,  6.2179e-02,  3.8479e-01, -2.2683e-03,  7.3421e-01,\n",
       "          1.9867e-01, -1.7005e-01, -5.8061e-01,  6.1885e-02, -4.4230e-01,\n",
       "         -7.4421e-02, -1.6320e-01, -5.0988e-02,  4.6794e-01,  4.5912e-01,\n",
       "         -1.5221e-01,  9.3750e-02, -1.7329e-01,  9.6182e-02, -4.0823e-01,\n",
       "          6.4527e-02, -3.6043e-01,  1.3212e-01, -2.6099e-01, -2.9154e-01,\n",
       "         -3.7432e-01,  2.8719e-01,  4.3891e-01,  2.6618e-01, -4.8076e-01,\n",
       "          6.4000e-01,  1.9996e-01, -1.7026e-01, -2.2563e-01, -3.8437e-02,\n",
       "          3.4650e-01,  3.0922e-01,  1.1747e-03,  6.6841e-01,  3.4594e-01,\n",
       "          1.9895e-01,  4.0211e-01, -2.0808e-01,  4.0794e-01, -5.1026e-01,\n",
       "          2.5054e-02,  2.3827e-01,  7.6360e-01,  2.0292e-01, -3.3808e-02,\n",
       "          3.9786e-01,  4.2702e-01, -1.9901e-01, -1.5481e-01,  2.0363e-01,\n",
       "         -4.6784e-01, -1.6299e-01,  8.5943e-01, -1.2681e-01,  2.9046e-01,\n",
       "         -5.9157e-02, -3.4299e-03, -2.9384e-01,  5.7925e-03,  6.3568e-01,\n",
       "          3.2342e-01, -1.0298e-01, -9.6554e-03,  4.5648e-01,  1.9576e-01,\n",
       "          1.0203e-02,  1.2445e-01, -3.4769e-01,  1.1490e-01, -9.9252e-02,\n",
       "         -6.1753e-01, -4.6705e-02, -2.7055e-01,  1.6090e-01, -3.4086e-01,\n",
       "         -2.1100e-01,  2.2879e-02, -2.1669e-02,  9.0677e-01, -2.8190e-01,\n",
       "         -2.0361e-01, -2.1859e-01, -1.6187e-01,  5.0468e-02, -6.1477e-02,\n",
       "         -3.1121e-01,  4.5543e-01,  1.8019e-01, -3.3103e-01,  3.9696e-01,\n",
       "          1.3047e-01, -1.6894e-02, -6.3304e-01, -4.3492e-01, -2.0265e-01,\n",
       "          2.8370e-01, -6.2301e-01, -1.4130e-01,  2.3103e-01,  5.1775e-02,\n",
       "         -2.8722e-01,  1.6709e-01,  3.1409e-01,  3.4725e-01, -4.5087e-01,\n",
       "          1.0540e-01,  1.9273e-01, -1.9792e-01, -2.0507e-01,  1.6554e-01,\n",
       "         -3.4340e-01, -5.6811e-02, -3.1969e-01,  3.8546e-01,  5.3657e-02,\n",
       "         -3.8670e-01,  7.9143e-02, -4.1264e-02, -4.3146e-01,  1.1572e-01,\n",
       "          2.8666e-01,  1.9627e-01, -2.8746e-01,  1.2080e-01,  4.7748e-01,\n",
       "          5.1259e-01, -4.7403e-01, -3.9579e-01, -7.6639e-02, -5.7779e-01,\n",
       "         -2.2547e-01,  2.7752e-01,  5.8264e-01, -5.7480e-02,  4.0947e-02,\n",
       "         -1.2603e-01,  1.5001e-01,  6.4848e-01, -1.2557e-01, -2.9649e-01,\n",
       "          2.9064e-01, -1.0008e-02, -2.4099e-01, -4.2563e-01, -2.3985e-01,\n",
       "         -3.4146e-01,  1.5939e-01,  2.9826e-01,  2.9757e-01,  5.2335e-02,\n",
       "         -2.3776e-01, -8.4225e-02,  2.9285e-01,  4.2382e-01, -1.4796e-02,\n",
       "          4.9129e-01, -1.9886e-01, -1.7847e-01, -3.3629e-01,  6.2977e-01,\n",
       "          7.6284e-02, -1.5963e-01,  2.9345e-01,  7.2709e-02,  3.9091e-01,\n",
       "          6.4743e-02, -1.2749e-01,  1.5374e-01,  3.5810e-01,  3.0659e-01,\n",
       "         -3.5905e-02,  1.3783e-01, -1.2210e-01, -1.2981e-02,  4.3598e-02,\n",
       "         -3.9952e-02, -9.9538e-03,  1.4790e-01, -3.6253e-01, -3.7661e-01,\n",
       "         -4.9125e-01, -2.0833e-01,  1.2803e-01,  4.0057e-01, -1.4522e+00,\n",
       "          2.2986e-01, -1.2181e-02,  5.0004e-03, -3.0416e-01, -3.2663e-02,\n",
       "         -7.8545e-02,  5.5936e-01,  1.4404e-01,  1.7029e-01,  2.6897e-01,\n",
       "          1.8055e-01, -2.6593e-01, -6.8670e-02,  3.1931e-01, -1.0739e-01,\n",
       "         -1.6615e-01, -1.3195e-01, -1.7583e-02, -2.0275e-01, -1.0056e-01,\n",
       "          3.1574e-01,  3.0659e-01, -9.5307e-02,  5.4824e-01, -3.0954e-01,\n",
       "          1.2739e-01, -1.0848e+00,  4.0507e-01, -1.3731e-01, -1.8255e-01,\n",
       "          2.9596e-01, -9.6700e-02,  4.2538e-02, -1.2628e-02, -2.2608e-01,\n",
       "         -3.5955e-01,  1.9460e-01,  2.7941e-01,  3.4903e-02,  1.3991e-01,\n",
       "          3.5938e-01,  5.5415e-02, -1.8385e-01, -6.5064e-02,  1.2899e-01,\n",
       "         -6.7445e-02, -4.8152e-01,  2.0789e-01,  1.8352e-01, -2.3852e-01,\n",
       "          8.8490e-02, -5.1522e-02, -2.8353e-01, -6.0435e-02, -3.0925e-01,\n",
       "         -1.1575e-01, -3.9677e-01,  8.2189e-03, -1.3074e-01,  8.1732e-02,\n",
       "         -3.8318e-01, -3.6953e-01,  3.1585e-01,  2.9878e-01, -9.8857e-02,\n",
       "         -8.5406e-02,  9.1839e-02, -2.1256e-01, -2.4259e-01, -4.7356e-02,\n",
       "         -3.4465e-01, -1.9922e-01,  1.6666e-01,  6.0882e-01, -3.7288e-02,\n",
       "         -1.7837e-01,  2.8214e-01, -3.3454e-01,  6.3730e-01, -1.4872e-01,\n",
       "         -1.2339e-01, -9.1321e-02, -1.9469e-01, -6.8055e-01,  4.1583e-01,\n",
       "         -1.5728e-01, -7.6475e-02,  1.7562e-01,  6.1624e-01,  3.5565e-01,\n",
       "          1.2243e-01, -2.6549e-02, -2.0167e-01,  3.9499e-01,  3.8333e-02,\n",
       "          7.5015e-02,  1.1153e+00, -1.7146e-01, -1.3666e-01, -1.1085e-01,\n",
       "          3.4161e-01, -1.3215e-01,  1.3778e-01, -2.5397e-01,  2.2227e-02,\n",
       "          9.2802e-02, -1.3949e-01,  5.3311e-01, -5.9121e-01, -2.6836e-01,\n",
       "          1.4797e-01, -1.8811e-01,  4.0979e-01, -6.3634e-02, -1.4555e-01,\n",
       "          2.6140e-01,  3.0455e-01,  2.6667e-01,  1.0222e-02, -4.9281e-02,\n",
       "         -1.8737e-01, -9.6869e-01,  2.4703e-01, -1.3484e-02,  2.8726e-02,\n",
       "         -1.8272e-02,  2.0853e-01, -5.1086e-02, -5.9549e-02, -3.5415e-01,\n",
       "         -2.9266e-01, -1.6444e-01, -3.3937e-02, -3.4207e-01,  6.6547e-01,\n",
       "         -5.3857e-02,  2.7065e-01, -6.1936e-01, -3.6081e-01,  3.9194e-01,\n",
       "         -2.9385e-02, -1.3917e-01,  8.3737e-02,  1.0611e+00,  1.6074e-02,\n",
       "         -1.1532e-01, -2.7015e-01,  5.0692e-01,  2.9221e-02,  1.7549e-01,\n",
       "          5.8699e-02, -9.9447e-02,  3.9572e-01,  5.0310e-01, -5.3736e-01,\n",
       "          3.6705e-02,  3.7710e-01,  2.7874e-01, -7.6413e-03, -1.6177e-01,\n",
       "         -2.1182e-02, -9.4384e-03,  2.9041e-02, -3.4159e-01,  1.5876e-01,\n",
       "          2.9884e-01,  7.1047e-02,  3.2576e-01,  5.1623e-01, -1.6385e-01,\n",
       "          2.8455e-01,  3.2378e-01,  2.2051e-01, -3.0408e-01, -4.8168e-01,\n",
       "          1.0075e-01,  3.4293e-01,  3.1962e-01,  2.8992e-01,  4.4777e-01,\n",
       "          6.8341e-01, -3.8050e-01, -1.5749e-01, -5.4223e-02, -4.1968e-02,\n",
       "          2.6355e-01,  2.8246e-01,  1.4440e-01, -6.8824e-01, -3.4200e-01,\n",
       "          1.8117e-01, -4.1018e-01,  1.5798e-02, -3.8522e-01, -4.5454e-01,\n",
       "         -6.2789e-02,  1.2112e-01, -2.7023e-01,  2.5691e-01, -1.4054e-01,\n",
       "          7.1012e-02, -1.2375e-02,  1.5621e-01, -3.5720e-01,  3.3962e-01,\n",
       "          2.4032e-02,  3.0787e-01,  4.4187e-02, -1.5740e-01,  8.0934e-02,\n",
       "         -3.4215e-02,  1.9108e-02, -8.8326e-01, -2.7611e-02,  3.3548e-01,\n",
       "         -5.2881e-02,  2.1000e-02, -2.3822e-01, -5.2379e-02, -5.1745e-01,\n",
       "         -5.5718e-02, -2.3807e-01, -2.4918e-01,  1.5981e-01, -7.6645e-01,\n",
       "         -1.7168e-01,  3.2980e-01,  2.8142e-01, -1.9060e-01,  4.7565e-01,\n",
       "         -2.8181e-01, -2.0520e-01, -1.5858e-01,  2.0174e-01,  1.3797e-01,\n",
       "         -2.2969e-01,  6.9177e-01, -1.5639e-01, -4.3299e-01,  3.2998e-01,\n",
       "          4.0267e-02,  1.0519e-01, -5.0986e-01,  3.1619e-01, -1.3738e-01,\n",
       "          1.5260e-01,  3.4159e-01,  7.2816e-01, -1.0810e-01, -2.6631e-01,\n",
       "          4.6921e-01,  8.9216e-02,  2.6076e-01, -4.1730e-01,  2.3594e-01,\n",
       "         -1.5076e-01,  1.6330e-01,  3.6587e-01, -8.8919e-01,  1.3149e-02,\n",
       "         -1.9772e-01, -8.8182e-01, -1.0959e-01,  8.0698e-01,  6.9452e-01,\n",
       "         -3.4532e-01,  1.1246e-01, -1.8184e-01,  1.2622e-01, -2.1774e-01,\n",
       "         -1.4921e-02, -2.1963e-01,  4.3899e-01, -2.4531e-01,  7.9933e-02,\n",
       "         -2.8845e-01, -8.2844e-01, -1.3598e-01,  3.3594e-01, -4.0136e-01,\n",
       "         -1.2828e-01,  1.3238e-01,  5.5686e-02,  2.1532e-01,  2.2414e-01,\n",
       "         -3.0435e-01,  2.0259e-01, -3.4683e-02,  2.4041e-02, -9.8629e-02,\n",
       "          8.3441e-02, -1.3490e-01, -1.6664e-01, -1.0387e-01, -7.2240e-02,\n",
       "         -1.2710e-01, -2.1375e-01, -1.0359e-01,  1.1537e-01,  4.7579e-02,\n",
       "          3.4517e-01, -1.0966e-01, -4.5432e-01, -1.0649e-01, -1.0234e-01,\n",
       "         -9.6426e-02,  8.3932e-02, -1.0280e-01,  3.0763e-01,  1.2954e-01,\n",
       "         -8.3576e-02, -2.8063e-01, -2.0901e-01, -3.8675e-01,  1.7422e-01,\n",
       "          5.2833e-01,  1.0518e-01, -4.5100e-01,  2.4380e-01, -2.3581e-02,\n",
       "          3.1645e-01,  1.4098e-01, -5.6060e-01,  1.1616e-01,  2.6374e-01,\n",
       "         -3.1148e-01,  2.9678e-01, -9.0686e-02, -4.5002e-01, -1.0523e-01,\n",
       "         -1.3845e-01, -3.2581e-01, -1.0701e-02,  1.5117e-01, -4.1111e-02,\n",
       "         -3.5451e-03,  1.0156e-01, -3.0193e-01,  1.3743e-01, -7.1552e-02,\n",
       "          2.1767e-01, -8.3261e-02,  4.1303e-01, -4.6924e-01, -2.9840e-01,\n",
       "          1.4235e-01,  1.7499e-01,  2.5991e-01,  1.4936e-01,  3.5778e-02,\n",
       "          7.0628e-02, -4.3159e-01,  4.2154e-01,  3.7768e-01, -3.1587e-01,\n",
       "         -5.2949e-02, -2.7423e-01, -2.0619e-01,  4.1262e-01,  2.1205e-01,\n",
       "         -8.6177e-02, -3.8261e-02, -1.8199e-01,  3.9677e-01, -3.5723e-01,\n",
       "          3.3150e-01, -1.2857e-02, -6.0813e-02, -8.1316e-01, -4.0352e-01,\n",
       "          1.3397e-01, -2.0389e-01,  3.9813e-04, -5.8109e-01, -5.4017e-01,\n",
       "         -1.4299e-01, -2.0194e-01, -4.3928e-01, -7.9109e-02, -2.9002e-01,\n",
       "          1.6643e-02, -4.5106e-01,  1.3754e-01,  5.2305e-01,  3.2817e-01,\n",
       "         -2.2742e-01, -2.3273e-01,  1.6968e-01, -2.5237e-01,  2.5388e-01,\n",
       "          1.2913e-01, -3.3585e-02,  4.7344e-02,  4.9046e-01,  1.4321e-01,\n",
       "          1.2433e-01, -4.2165e-01,  8.5115e-03, -2.9829e-02,  2.6215e-02,\n",
       "          1.9132e-01, -3.1367e-01,  1.1619e-02,  3.3291e-03,  2.3753e-01,\n",
       "         -1.2109e-01, -1.1076e-01,  3.0016e-01, -2.0713e-01,  6.2541e-02,\n",
       "          2.9938e-02,  1.1100e-01,  2.5003e-01,  5.4072e-02,  2.3931e-02,\n",
       "          8.7415e-01, -1.1659e-01,  3.1650e-01, -8.1356e-02,  8.9519e-02,\n",
       "          7.3017e-01,  1.3797e-03,  1.9707e-02, -5.3940e-01,  5.7334e-01,\n",
       "         -5.6615e-02,  5.1498e-01, -3.5614e-01, -2.6152e-01, -3.1733e-02,\n",
       "          2.0266e-01, -4.8242e-01, -2.0163e-01,  8.8836e-02,  6.9157e-02,\n",
       "         -2.6534e-01,  9.9821e-02,  5.5912e-02,  7.6970e-01,  6.4160e-01,\n",
       "         -1.7163e-01,  1.2468e-01, -2.4122e-01,  2.3969e-01,  3.5857e-01,\n",
       "          1.9039e-02,  8.1772e-02,  3.3391e-01, -6.9354e-02,  6.9029e-02,\n",
       "         -1.2088e-01, -1.0914e-01, -2.1385e-01,  1.9172e-02,  5.2380e-02,\n",
       "          4.5071e-01,  1.1229e-01,  1.2006e-01,  5.6065e-03, -6.5407e-01,\n",
       "         -4.8379e-03, -2.2309e-01,  8.3660e-02, -6.8155e-02,  5.8539e-01,\n",
       "          1.9238e-02, -6.9104e-01,  1.5199e-01, -2.2915e-01, -6.2103e-02,\n",
       "          3.0751e-01,  2.3734e-01, -4.4469e-01, -7.5367e-02, -6.5961e-02,\n",
       "          2.1845e-01, -2.4769e-01,  6.0343e-02, -3.4305e-02,  1.2984e-01,\n",
       "         -5.7548e-03,  1.1150e-02, -1.2808e-01, -2.1543e-01,  2.1532e-01,\n",
       "         -1.6476e-01, -9.4801e-02, -4.5693e-01,  4.0483e-01,  6.0412e-01,\n",
       "         -2.0267e-01,  3.4277e-01, -2.3517e-01, -9.9057e-02, -5.2509e-01,\n",
       "          1.4286e-01, -1.5221e+00,  8.6342e-02, -2.6853e-02,  9.5929e-01,\n",
       "         -2.6923e-01,  1.6916e-01,  5.1306e-01, -3.3715e-01,  1.9992e-01,\n",
       "          1.3118e-01, -2.0869e-01, -2.0449e-01,  9.6120e-02,  1.2016e-02,\n",
       "          4.2820e-01, -1.0336e-01, -4.6412e-02, -6.1318e-02, -3.7152e-01,\n",
       "         -1.2900e-01,  9.6740e-03, -2.0775e-01, -1.7015e-01, -4.3225e-02,\n",
       "         -2.0433e-01, -8.9574e-03,  3.7457e-01,  7.9207e-02,  7.2365e-01,\n",
       "         -7.3823e-01, -5.9392e-01, -1.1875e-01, -1.5766e-01,  1.8748e-01,\n",
       "          5.5474e-02,  1.3120e-01, -4.9233e-01]], grad_fn=<MeanBackward1>)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.2685, -0.5767,  0.1836,  ..., -0.0248,  0.0143, -0.3322],\n",
       "         [-0.1389, -0.2711, -0.1331,  ...,  0.1358,  0.2481, -0.6525]]],\n",
       "       grad_fn=<NativeLayerNormBackward>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'tuple' object has no attribute 'get_output_embeddings'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-829d38ea8a32>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_output_embeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute 'get_output_embeddings'"
     ]
    }
   ],
   "source": [
    "model(input_ids).get_output_embeddings()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}