datasets/dataset_scientific.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "dataset-scientific.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c8548d6bac70415296f733c4fa0544e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_673e0707256441afb47831aefbf309a1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7cb878f1d83844498809ee2897584167",
              "IPY_MODEL_91b7573eef16458782ed1918acb54885",
              "IPY_MODEL_21d8875481bb47e1b09d7d65523bbcb5"
            ]
          }
        },
        "673e0707256441afb47831aefbf309a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7cb878f1d83844498809ee2897584167": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_791817222188418ea74331641ae32f7a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "/content/datasets/trec-covid.zip: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_70f15618819b4ccb9c8f78f5b1e913ee"
          }
        },
        "91b7573eef16458782ed1918acb54885": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4e72346e05ad4370a4366e217903f926",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 73876720,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 73876720,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e7a31feed4940a58ac7364ba1c732bc"
          }
        },
        "21d8875481bb47e1b09d7d65523bbcb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_61793029235c4919a3a13b1d5c201506",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 70.5M/70.5M [00:06&lt;00:00, 21.5MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_20d61a22704d47cb829ea8862244e649"
          }
        },
        "791817222188418ea74331641ae32f7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "70f15618819b4ccb9c8f78f5b1e913ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e72346e05ad4370a4366e217903f926": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e7a31feed4940a58ac7364ba1c732bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61793029235c4919a3a13b1d5c201506": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "20d61a22704d47cb829ea8862244e649": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a56046d90148429082a4a41134809697": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_31b4843c3f9842548bd7421094b5111f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c478b0495bfb4bb584011c1ad09cdee5",
              "IPY_MODEL_3b185c4c8cb54694b2505ea1ab4c1d01",
              "IPY_MODEL_0198d47429d44bf1801b5d205357ea2c"
            ]
          }
        },
        "31b4843c3f9842548bd7421094b5111f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c478b0495bfb4bb584011c1ad09cdee5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bbdd18b38b3e424d8d0542ff70b07a8b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "/content/datasets/scifact.zip: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f04b1a61d85740f0bbcd69672fb63e97"
          }
        },
        "3b185c4c8cb54694b2505ea1ab4c1d01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9a778baddd234cd79dc24f186daed3db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2816079,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2816079,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c66819f650f4057974229695269e90b"
          }
        },
        "0198d47429d44bf1801b5d205357ea2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c8fb3ca609d4443895cc59336a966ea5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.69M/2.69M [00:02&lt;00:00, 2.61MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83328b08cd024df5a07d669fe38aa0c6"
          }
        },
        "bbdd18b38b3e424d8d0542ff70b07a8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f04b1a61d85740f0bbcd69672fb63e97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a778baddd234cd79dc24f186daed3db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c66819f650f4057974229695269e90b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8fb3ca609d4443895cc59336a966ea5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83328b08cd024df5a07d669fe38aa0c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ecdd787ef2b647a29a24c0023fe44ce8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bf01e427159c42ca9e136ec7d682ceb5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c2c4d5c1dba44d22a2aaf1811ea511d6",
              "IPY_MODEL_4283666ee54a487bb43a4db542419f89",
              "IPY_MODEL_ff3b7f9ab10449c6922c5d28d96711c7"
            ]
          }
        },
        "bf01e427159c42ca9e136ec7d682ceb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c2c4d5c1dba44d22a2aaf1811ea511d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_198abad54d9d4413881147b4a1e5fa4b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "/content/datasets/scidocs.zip: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3465c85abbfa482eb06bb1ea05455fa3"
          }
        },
        "4283666ee54a487bb43a4db542419f89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8a511f3756524bed9c255bc77178bfc7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 142471588,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 142471588,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0d982a38037a4dfa9ceacda5970c28cf"
          }
        },
        "ff3b7f9ab10449c6922c5d28d96711c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_03e24bafdd534794a22fc4e73667e09c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 136M/136M [00:08&lt;00:00, 29.1MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22701ede29524c2a8301e06ac91f8b86"
          }
        },
        "198abad54d9d4413881147b4a1e5fa4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3465c85abbfa482eb06bb1ea05455fa3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a511f3756524bed9c255bc77178bfc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0d982a38037a4dfa9ceacda5970c28cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "03e24bafdd534794a22fc4e73667e09c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22701ede29524c2a8301e06ac91f8b86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9fad279da2a74b20944fb3c17914f7d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_818e80036dff493592a2d270b27b076f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4781b96dd87e4135b9dba286212138a1",
              "IPY_MODEL_f2aae1e482e54846b1f8d2768f6d61d2",
              "IPY_MODEL_2b37826bbf4b4ecdbf7417879a73b661"
            ]
          }
        },
        "818e80036dff493592a2d270b27b076f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4781b96dd87e4135b9dba286212138a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a69a188b537346038a8fdd2a138b100a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "/content/datasets/nfcorpus.zip: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_06677fdd8f274fb5af07be4b8a8a6dbf"
          }
        },
        "f2aae1e482e54846b1f8d2768f6d61d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7f29838c5989401499099414dde77f58",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2448432,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2448432,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_836b0e509ea1470fa632a1fc5da1025e"
          }
        },
        "2b37826bbf4b4ecdbf7417879a73b661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7177cb875ecc4fb7b8cb3defcdea4575",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.34M/2.34M [00:01&lt;00:00, 2.34MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4bec0dfe7a5042eca7b0be8c23b35271"
          }
        },
        "a69a188b537346038a8fdd2a138b100a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "06677fdd8f274fb5af07be4b8a8a6dbf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7f29838c5989401499099414dde77f58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "836b0e509ea1470fa632a1fc5da1025e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7177cb875ecc4fb7b8cb3defcdea4575": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4bec0dfe7a5042eca7b0be8c23b35271": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KbTUYoH4FeAf"
      },
      "source": [
        "DATASETS"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MMS-ANL3D7vd",
        "outputId": "71853a92-5461-4b45-93a1-d7eea7eecb54"
      },
      "source": [
        "!pip install -U beir\n",
        "!pip install nltk\n",
        "!pip install pandas"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting beir\n",
            "  Downloading beir-0.2.3.tar.gz (52 kB)\n",
            "\u001b[K     |████████████████████████████████| 52 kB 945 kB/s \n",
            "\u001b[?25hCollecting sentence-transformers\n",
            "  Downloading sentence-transformers-2.1.0.tar.gz (78 kB)\n",
            "\u001b[K     |████████████████████████████████| 78 kB 4.9 MB/s \n",
            "\u001b[?25hCollecting pytrec_eval\n",
            "  Downloading pytrec_eval-0.5.tar.gz (15 kB)\n",
            "Collecting faiss_cpu\n",
            "  Downloading faiss_cpu-1.7.1.post2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 8.4 MB 5.8 MB/s \n",
            "\u001b[?25hCollecting elasticsearch\n",
            "  Downloading elasticsearch-7.15.1-py2.py3-none-any.whl (378 kB)\n",
            "\u001b[K     |████████████████████████████████| 378 kB 52.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from beir) (2.6.0)\n",
            "Collecting tensorflow-text\n",
            "  Downloading tensorflow_text-2.6.0-cp37-cp37m-manylinux1_x86_64.whl (4.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 56.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow-hub in /usr/local/lib/python3.7/dist-packages (from beir) (0.12.0)\n",
            "Requirement already satisfied: tensorflow-estimator~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (2.6.0)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.12)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (0.12.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.1.0)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (0.37.0)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.15.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.19.5)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.12.1)\n",
            "Requirement already satisfied: keras~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (2.6.0)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.6.3)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (0.4.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (0.2.0)\n",
            "Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (2.6.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (3.7.4.3)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.1.2)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (3.1.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (1.41.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (3.3.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (3.17.3)\n",
            "Requirement already satisfied: clang~=5.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.2.0->beir) (5.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow>=2.2.0->beir) (1.5.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (3.3.4)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (57.4.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (1.35.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (1.8.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow>=2.2.0->beir) (1.0.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->beir) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->beir) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->beir) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2.2.0->beir) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2.2.0->beir) (4.8.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow>=2.2.0->beir) (0.4.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->beir) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->beir) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->beir) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow>=2.2.0->beir) (2021.5.30)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow>=2.2.0->beir) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.6->tensorflow>=2.2.0->beir) (3.6.0)\n",
            "Collecting transformers<5.0.0,>=4.6.0\n",
            "  Downloading transformers-4.12.3-py3-none-any.whl (3.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1 MB 41.2 MB/s \n",
            "\u001b[?25hCollecting tokenizers>=0.10.3\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 43.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->beir) (4.62.3)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->beir) (1.9.0+cu111)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->beir) (0.10.0+cu111)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->beir) (0.22.2.post1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->beir) (1.4.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from sentence-transformers->beir) (3.2.5)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 50.7 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub\n",
            "  Downloading huggingface_hub-0.1.0-py3-none-any.whl (59 kB)\n",
            "\u001b[K     |████████████████████████████████| 59 kB 6.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->beir) (21.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 67.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->beir) (2019.12.20)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 53.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->beir) (3.3.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers<5.0.0,>=4.6.0->sentence-transformers->beir) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<5.0.0,>=4.6.0->sentence-transformers->beir) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<5.0.0,>=4.6.0->sentence-transformers->beir) (1.0.1)\n",
            "Requirement already satisfied: pillow>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision->sentence-transformers->beir) (7.1.2)\n",
            "Building wheels for collected packages: beir, pytrec-eval, sentence-transformers\n",
            "  Building wheel for beir (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for beir: filename=beir-0.2.3-py3-none-any.whl size=45805 sha256=ba749df672ee7267f6f23cecfc15b33dae1391e800998b5cedceba2b6fdbb31e\n",
            "  Stored in directory: /root/.cache/pip/wheels/af/29/4c/261c842753d34921b8b35c1b8679720114f8536654767430dd\n",
            "  Building wheel for pytrec-eval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytrec-eval: filename=pytrec_eval-0.5-cp37-cp37m-linux_x86_64.whl size=264595 sha256=331a6dc6a9e375ee666639977a386b662a80aa3c43acef923aaae57d409c4c3a\n",
            "  Stored in directory: /root/.cache/pip/wheels/42/96/77/0829b8b2606f90f61ba10a51277629d2b615604e122ee932f4\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.1.0-py3-none-any.whl size=121000 sha256=2b8f28ea14ec5666cc056f8eeb3db9cacaf154255a4736329e8a84aa0849446e\n",
            "  Stored in directory: /root/.cache/pip/wheels/90/f0/bb/ed1add84da70092ea526466eadc2bfb197c4bcb8d4fa5f7bad\n",
            "Successfully built beir pytrec-eval sentence-transformers\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers, sentencepiece, tensorflow-text, sentence-transformers, pytrec-eval, faiss-cpu, elasticsearch, beir\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed beir-0.2.3 elasticsearch-7.15.1 faiss-cpu-1.7.1.post2 huggingface-hub-0.1.0 pytrec-eval-0.5 pyyaml-6.0 sacremoses-0.0.46 sentence-transformers-2.1.0 sentencepiece-0.1.96 tensorflow-text-2.6.0 tokenizers-0.10.3 transformers-4.12.3\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk) (1.15.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.19.5)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPHS4YFuFoma",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "75f53cb1-42a6-4bc6-f844-eede291b72c1"
      },
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "from nltk.probability import FreqDist\n",
        "from nltk.corpus import stopwords\n",
        "from beir import util"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/beir/util.py:2: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kh8g3a0PGSz-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cccb0111-4c96-4626-a98e-f24cb335cb35"
      },
      "source": [
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "stop_words = set(stopwords.words('english'))\n",
        "pd.set_option('display.max_columns', None)\n",
        "pd.set_option('display.max_rows', None)\n",
        "dataset_url = 'https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip'"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8qPjo9Rbl0FU"
      },
      "source": [
        "def load_dataset(dataset_name):\n",
        "    out_dir = os.path.join(os.getcwd(), 'datasets')\n",
        "    util.download_and_unzip(dataset_url.format(dataset_name), out_dir)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fgd-5d6SmFo_"
      },
      "source": [
        "def filter_words(w, filter_stopwords):\n",
        "    if filter_stopwords:\n",
        "        return w.isalnum() and w not in stop_words\n",
        "    else:\n",
        "        return w"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "faV98j-Ifw5v"
      },
      "source": [
        "def build_doc_stats(dataset_name, fdist, doc_count, sentence_count, query_count):\n",
        "    term_count = fdist.N()\n",
        "    bin_count = fdist.B()\n",
        "    word_per_doc = term_count / doc_count\n",
        "    valid_sentence_count = sentence_count\n",
        "    if valid_sentence_count == 0:\n",
        "        print('WARNING! \"{}\" corpus does not have title values.'.format(dataset_name))\n",
        "        valid_sentence_count = 1\n",
        "    word_per_sent = term_count / valid_sentence_count\n",
        "    # print(fdist.most_common(10))\n",
        "    stats_dict = {\n",
        "        '# of docs': [doc_count],\n",
        "        '# of terms': [term_count],\n",
        "        '# of unique terms': [bin_count],\n",
        "        '# of sentences': [sentence_count],\n",
        "        '# of terms per sentences': [word_per_sent],\n",
        "        '# of terms per docs': [word_per_doc]}\n",
        "    if query_count != 0:\n",
        "        stats_dict['# of queries'] = query_count\n",
        "    stats_df = pd.DataFrame(data=stats_dict)\n",
        "    stats_df.index = [dataset_name]\n",
        "    return stats_df"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qluVCUB2gIAN"
      },
      "source": [
        "def get_corpus_df(dataset_name, file_name):\n",
        "    corpus_path = os.path.join(os.getcwd(), 'datasets', dataset_name, file_name)\n",
        "    return pd.read_json(path_or_buf=corpus_path, lines=True)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nuO6Se4svb8d"
      },
      "source": [
        "def stats_corpus(dataset_name, column_name, add_queries):\n",
        "    load_dataset(dataset_name)\n",
        "    query_count = 0\n",
        "    if add_queries:\n",
        "        queries_df = get_corpus_df(dataset_name, 'queries.jsonl')\n",
        "        query_count = len(queries_df.index)\n",
        "    corpus_df = get_corpus_df(dataset_name, 'corpus.jsonl')\n",
        "    doc_count = len(corpus_df.index)\n",
        "    sentence_count = 0\n",
        "    fdist = FreqDist()\n",
        "    for line in corpus_df.get(column_name):\n",
        "        sentence_count += len(sent_tokenize(line))\n",
        "        tokens = [w.lower() for w in word_tokenize(line) if filter_words(w, True)]\n",
        "        for word in tokens:\n",
        "            fdist[word] += 1\n",
        "    return build_doc_stats(dataset_name, fdist, doc_count, sentence_count, query_count)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uej16KOhmTCP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "c8548d6bac70415296f733c4fa0544e8",
            "673e0707256441afb47831aefbf309a1",
            "7cb878f1d83844498809ee2897584167",
            "91b7573eef16458782ed1918acb54885",
            "21d8875481bb47e1b09d7d65523bbcb5",
            "791817222188418ea74331641ae32f7a",
            "70f15618819b4ccb9c8f78f5b1e913ee",
            "4e72346e05ad4370a4366e217903f926",
            "0e7a31feed4940a58ac7364ba1c732bc",
            "61793029235c4919a3a13b1d5c201506",
            "20d61a22704d47cb829ea8862244e649",
            "a56046d90148429082a4a41134809697",
            "31b4843c3f9842548bd7421094b5111f",
            "c478b0495bfb4bb584011c1ad09cdee5",
            "3b185c4c8cb54694b2505ea1ab4c1d01",
            "0198d47429d44bf1801b5d205357ea2c",
            "bbdd18b38b3e424d8d0542ff70b07a8b",
            "f04b1a61d85740f0bbcd69672fb63e97",
            "9a778baddd234cd79dc24f186daed3db",
            "4c66819f650f4057974229695269e90b",
            "c8fb3ca609d4443895cc59336a966ea5",
            "83328b08cd024df5a07d669fe38aa0c6",
            "ecdd787ef2b647a29a24c0023fe44ce8",
            "bf01e427159c42ca9e136ec7d682ceb5",
            "c2c4d5c1dba44d22a2aaf1811ea511d6",
            "4283666ee54a487bb43a4db542419f89",
            "ff3b7f9ab10449c6922c5d28d96711c7",
            "198abad54d9d4413881147b4a1e5fa4b",
            "3465c85abbfa482eb06bb1ea05455fa3",
            "8a511f3756524bed9c255bc77178bfc7",
            "0d982a38037a4dfa9ceacda5970c28cf",
            "03e24bafdd534794a22fc4e73667e09c",
            "22701ede29524c2a8301e06ac91f8b86",
            "9fad279da2a74b20944fb3c17914f7d8",
            "818e80036dff493592a2d270b27b076f",
            "4781b96dd87e4135b9dba286212138a1",
            "f2aae1e482e54846b1f8d2768f6d61d2",
            "2b37826bbf4b4ecdbf7417879a73b661",
            "a69a188b537346038a8fdd2a138b100a",
            "06677fdd8f274fb5af07be4b8a8a6dbf",
            "7f29838c5989401499099414dde77f58",
            "836b0e509ea1470fa632a1fc5da1025e",
            "7177cb875ecc4fb7b8cb3defcdea4575",
            "4bec0dfe7a5042eca7b0be8c23b35271"
          ]
        },
        "outputId": "87564680-0c4c-449f-fb17-f6f9e7ae68e0"
      },
      "source": [
        "datasets = ['trec-covid', 'scifact', 'scidocs', 'nfcorpus']\n",
        "title_stats = []\n",
        "text_stats = []\n",
        "for dataset in datasets:\n",
        "    title_stats.append(stats_corpus(dataset, 'title', False))\n",
        "for dataset in datasets:\n",
        "    text_stats.append(stats_corpus(dataset, 'text', True))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c8548d6bac70415296f733c4fa0544e8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "/content/datasets/trec-covid.zip:   0%|          | 0.00/70.5M [00:00<?, ?iB/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a56046d90148429082a4a41134809697",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "/content/datasets/scifact.zip:   0%|          | 0.00/2.69M [00:00<?, ?iB/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ecdd787ef2b647a29a24c0023fe44ce8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "/content/datasets/scidocs.zip:   0%|          | 0.00/136M [00:00<?, ?iB/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9fad279da2a74b20944fb3c17914f7d8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "/content/datasets/nfcorpus.zip:   0%|          | 0.00/2.34M [00:00<?, ?iB/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m2SN5AlUvR9d"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ccIWU-yW7TB"
      },
      "source": [
        "**TREC-COVID** - [Homepage](https://ir.nist.gov/covidSubmit/index.html) - [Paper](https://arxiv.org/abs/2005.04474)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eFpAGDZzp1ju"
      },
      "source": [
        "**SCIFACT** - [Homepage](https://allenai.org/data/scifact) - [Paper](https://arxiv.org/abs/2004.14974)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BlIW24wXW8GZ"
      },
      "source": [
        "**SCIDOCS** - [Homepage](https://allenai.org/data/scidocs) - [Paper](https://arxiv.org/abs/2004.07180)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_0cW7cQbW9Ci"
      },
      "source": [
        "**NFCORPUS** - [Homepage](https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/) - [Paper](https://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gYhstvyrvS-s"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3dY192rUvUd8"
      },
      "source": [
        "**STATISTICS FOR TITLES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "jiibaVjDg9Ba",
        "outputId": "c91ed153-86a0-484f-b05f-872bfcc49c20"
      },
      "source": [
        "pd.concat(title_stats)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th># of docs</th>\n",
              "      <th># of terms</th>\n",
              "      <th># of unique terms</th>\n",
              "      <th># of sentences</th>\n",
              "      <th># of terms per sentences</th>\n",
              "      <th># of terms per docs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>trec-covid</th>\n",
              "      <td>171332</td>\n",
              "      <td>1477906</td>\n",
              "      <td>61432</td>\n",
              "      <td>176580</td>\n",
              "      <td>8.369612</td>\n",
              "      <td>8.625978</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>scifact</th>\n",
              "      <td>5183</td>\n",
              "      <td>46307</td>\n",
              "      <td>9404</td>\n",
              "      <td>5291</td>\n",
              "      <td>8.752032</td>\n",
              "      <td>8.934401</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>scidocs</th>\n",
              "      <td>25657</td>\n",
              "      <td>181503</td>\n",
              "      <td>18755</td>\n",
              "      <td>26058</td>\n",
              "      <td>6.965347</td>\n",
              "      <td>7.074210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nfcorpus</th>\n",
              "      <td>3633</td>\n",
              "      <td>31914</td>\n",
              "      <td>6184</td>\n",
              "      <td>3763</td>\n",
              "      <td>8.480999</td>\n",
              "      <td>8.784476</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            # of docs  # of terms  # of unique terms  # of sentences  \\\n",
              "trec-covid     171332     1477906              61432          176580   \n",
              "scifact          5183       46307               9404            5291   \n",
              "scidocs         25657      181503              18755           26058   \n",
              "nfcorpus         3633       31914               6184            3763   \n",
              "\n",
              "            # of terms per sentences  # of terms per docs  \n",
              "trec-covid                  8.369612             8.625978  \n",
              "scifact                     8.752032             8.934401  \n",
              "scidocs                     6.965347             7.074210  \n",
              "nfcorpus                    8.480999             8.784476  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9-gHDhjPvgmV"
      },
      "source": [
        "**STATISTICS FOR DOCUMENTS**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "MpmravpSg9hT",
        "outputId": "164854b6-b0ed-4fac-d8d8-2247e1116e4d"
      },
      "source": [
        "pd.concat(text_stats)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th># of docs</th>\n",
              "      <th># of terms</th>\n",
              "      <th># of unique terms</th>\n",
              "      <th># of sentences</th>\n",
              "      <th># of terms per sentences</th>\n",
              "      <th># of terms per docs</th>\n",
              "      <th># of queries</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>trec-covid</th>\n",
              "      <td>171332</td>\n",
              "      <td>16053536</td>\n",
              "      <td>185434</td>\n",
              "      <td>1126955</td>\n",
              "      <td>14.245055</td>\n",
              "      <td>93.698410</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>scifact</th>\n",
              "      <td>5183</td>\n",
              "      <td>658662</td>\n",
              "      <td>32591</td>\n",
              "      <td>47121</td>\n",
              "      <td>13.978099</td>\n",
              "      <td>127.081227</td>\n",
              "      <td>1109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>scidocs</th>\n",
              "      <td>25657</td>\n",
              "      <td>2686311</td>\n",
              "      <td>71106</td>\n",
              "      <td>195663</td>\n",
              "      <td>13.729274</td>\n",
              "      <td>104.700900</td>\n",
              "      <td>1000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nfcorpus</th>\n",
              "      <td>3633</td>\n",
              "      <td>504134</td>\n",
              "      <td>24388</td>\n",
              "      <td>35130</td>\n",
              "      <td>14.350527</td>\n",
              "      <td>138.765208</td>\n",
              "      <td>3237</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            # of docs  # of terms  # of unique terms  # of sentences  \\\n",
              "trec-covid     171332    16053536             185434         1126955   \n",
              "scifact          5183      658662              32591           47121   \n",
              "scidocs         25657     2686311              71106          195663   \n",
              "nfcorpus         3633      504134              24388           35130   \n",
              "\n",
              "            # of terms per sentences  # of terms per docs  # of queries  \n",
              "trec-covid                 14.245055            93.698410            50  \n",
              "scifact                    13.978099           127.081227          1109  \n",
              "scidocs                    13.729274           104.700900          1000  \n",
              "nfcorpus                   14.350527           138.765208          3237  "
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k21nEMNOxPEM"
      },
      "source": [
        "end of fun."
      ]
    }
  ]
}