0_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run scripts/HltSamples.py\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate code bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!g++ scripts/generate_csv.cc -I /usr/local/include/root/ -L /usr/local/lib/root/ -lTree -lCore -lPhysics -lRIO -o scripts/generate_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!g++ scripts/skim.cc -I /usr/local/include/root/ -L /usr/local/lib/root/ -lTree -lCore -lPhysics -lRIO -o scripts/skim  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!mkdir datasets/skims"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!mkdir datasets/prepared_hlt_track"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!mkdir datasets/prepared_hlt_body"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 40 ms, sys: 84 ms, total: 124 ms\n",
      "Wall time: 13min 39s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open(\"skim.log\", \"w\") as l:\n",
    "    threads = []\n",
    "    for ch,val in Samples.iteritems():\n",
    "        threads.append(subprocess.Popen([\"scripts/skim\", str(ch), str(val[\"mother\"]), str(len(val[\"children\"])),\n",
    "                                         \"datasets\"],\n",
    "                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE))\n",
    "\n",
    "    for thread in threads:\n",
    "        t,e = thread.communicate()\n",
    "        l.write(t)\n",
    "        l.write(\"\\n\")\n",
    "        l.write(e)\n",
    "        l.write(\"\\n\")\n",
    "\n",
    "with open(\"generate_hlt1.log\", \"w\") as l:\n",
    "    threads = []\n",
    "    for ch,val in Samples.iteritems():\n",
    "        threads.append(subprocess.Popen([\"scripts/generate_csv\", str(ch), str(val[\"mother\"]), str(len(val[\"children\"])),\n",
    "                                         \"datasets\"],\n",
    "                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE))\n",
    "\n",
    "    for thread in threads:\n",
    "        t,e = thread.communicate()\n",
    "        l.write(t)\n",
    "        l.write(\"\\n\")\n",
    "        l.write(e)\n",
    "        l.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mode: 13102412\r\n",
      "Events: 31938; No tracks: 0; No sv: 2844\r\n",
      "0.951844 0.592366 0.605705\r\n",
      "\r\n",
      "\r\n",
      "Mode: 11114001\r\n",
      "Events: 61102; No tracks: 0; No sv: 1732\r\n",
      "0.993159 0.880397 0.873196\r\n",
      "\r\n",
      "\r\n",
      "Mode: 11536011\r\n",
      "Events: 13893; No tracks: 0; No sv: 157\r\n",
      "0.995753 0.867775 0.899734\r\n",
      "\r\n",
      "\r\n",
      "Mode: 11102003\r\n",
      "Events: 22317; No tracks: 0; No sv: 982\r\n",
      "0.975758 0.772416 0.77022\r\n",
      "\r\n",
      "\r\n",
      "Mode: 12873002\r\n",
      "Events: 9480; No tracks: 0; No sv: 317\r\n",
      "0.991983 0.779536 0.79789\r\n",
      "\r\n",
      "\r\n",
      "Mode: 12875500\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.12875500.root\r\n",
      "\r\n",
      "Mode: 13246001\r\n",
      "Events: 37313; No tracks: 0; No sv: 838\r\n",
      "0.995685 0.910219 0.907485\r\n",
      "\r\n",
      "\r\n",
      "Mode: 25103000\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.25103000.root\r\n",
      "\r\n",
      "Mode: 25103110\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.25103110.root\r\n",
      "\r\n",
      "Mode: 12165106\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.12165106.root\r\n",
      "\r\n",
      "Mode: 12873432\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.12873432.root\r\n",
      "\r\n",
      "Mode: 11124001\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.11124001.root\r\n",
      "\r\n",
      "Mode: 13264021\r\n",
      "Events: 25165; No tracks: 0; No sv: 398\r\n",
      "0.995112 0.881303 0.901411\r\n",
      "\r\n",
      "\r\n",
      "Mode: 13104012\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.13104012.root\r\n",
      "\r\n",
      "Mode: 11874042\r\n",
      "Events: 3942; No tracks: 0; No sv: 97\r\n",
      "0.995941 0.80619 0.8379\r\n",
      "\r\n",
      "\r\n",
      "Mode: 13144020\r\n",
      "Events: 39278; No tracks: 0; No sv: 1119\r\n",
      "0.993228 0.882682 0.877769\r\n",
      "\r\n",
      "\r\n",
      "Mode: 11296013\r\n",
      "Events: 17149; No tracks: 0; No sv: 84\r\n",
      "0.998834 0.934982 0.949326\r\n",
      "\r\n",
      "\r\n",
      "Mode: 27163002\r\n",
      "Events: 848; No tracks: 0; No sv: 129\r\n",
      "0.913915 0.542453 0.520047\r\n",
      "\r\n",
      "\r\n",
      "Mode: 11874004\r\n",
      "Events: 27212; No tracks: 0; No sv: 580\r\n",
      "0.994671 0.827172 0.850728\r\n",
      "\r\n",
      "\r\n",
      "Mode: 20000000\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.20000000.root\r\n",
      "\r\n",
      "Mode: 11104121\r\n",
      "Events: 21960; No tracks: 0; No sv: 1361\r\n",
      "0.969262 0.63174 0.651503\r\n",
      "\r\n",
      "\r\n",
      "Mode: 23103110\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.23103110.root\r\n",
      "\r\n",
      "Mode: 13144001\r\n",
      "Events: 54266; No tracks: 0; No sv: 1379\r\n",
      "0.993937 0.894575 0.889028\r\n",
      "\r\n",
      "\r\n",
      "Mode: 12103035\r\n",
      "Events: 25250; No tracks: 0; No sv: 713\r\n",
      "0.98602 0.852911 0.855842\r\n",
      "\r\n",
      "\r\n",
      "Mode: 21263002\r\n",
      "Events: 8871; No tracks: 0; No sv: 836\r\n",
      "0.955022 0.654718 0.661143\r\n",
      "\r\n",
      "\r\n",
      "Mode: 13112001\r\n",
      "Events: 34736; No tracks: 0; No sv: 1325\r\n",
      "0.989377 0.831875 0.828391\r\n",
      "\r\n",
      "\r\n",
      "Mode: 30000000\r\n",
      "Events: 111306; No tracks: 786; No sv: 73271\r\n",
      "0.861211 0.116445 0.099222\r\n",
      "\r\n",
      "\r\n",
      "Mode: 15164001\r\n",
      "Events: 7130; No tracks: 0; No sv: 143\r\n",
      "0.992146 0.892286 0.891445\r\n",
      "\r\n",
      "\r\n",
      "Mode: 27265001\r\n",
      "Events: 871; No tracks: 0; No sv: 144\r\n",
      "0.9093 0.466131 0.439724\r\n",
      "\r\n",
      "\r\n",
      "Mode: 13774002\r\n",
      "Events: 12589; No tracks: 0; No sv: 349\r\n",
      "0.993169 0.820716 0.85114\r\n",
      "\r\n",
      "\r\n",
      "Mode: 13102201\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "\r\n",
      "Mode: 27163400\r\n",
      "Events: 380; No tracks: 0; No sv: 89\r\n",
      "0.876316 0.355263 0.35\r\n",
      "\r\n",
      "\r\n",
      "Mode: 12265042\r\n",
      "Events: 0; No tracks: 0; No sv: 0\r\n",
      "-nan -nan -nan\r\n",
      "\r\n",
      "Error in <TChain::LoadTree>: Cannot find tree with name data in file datasets/skims/small.12265042.root\r\n",
      "\r\n",
      "Mode: 13512010\r\n",
      "Events: 43379; No tracks: 0; No sv: 3056\r\n",
      "0.973075 0.65315 0.668042\r\n",
      "\r\n",
      "\r\n",
      "Mode: 12103121\r\n",
      "Events: 22180; No tracks: 0; No sv: 3517\r\n",
      "0.90239 0.103652 0.129666\r\n",
      "\r\n",
      "\r\n",
      "Mode: 11114101\r\n",
      "Events: 36257; No tracks: 0; No sv: 2246\r\n",
      "0.984693 0.677469 0.691039\r\n",
      "\r\n",
      "\r\n",
      "Mode: 15512011\r\n",
      "Events: 27799; No tracks: 0; No sv: 2047\r\n",
      "0.966042 0.634627 0.643656\r\n",
      "\r\n",
      "\r\n",
      "Mode: 27265101\r\n",
      "Events: 2192; No tracks: 0; No sv: 507\r\n",
      "0.855383 0.286953 0.26688\r\n",
      "\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "! cat generate_hlt1.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "!mkdir models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!mkdir bbdt_run2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}