Botnet-Bidirectional.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "9351d253-da67-4a9a-b328-4789d3a5bfc8"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This non-commercial license of GraphLab Create for academic use is assigned to himadri.mishra.cse13@iitbhu.ac.in and will expire on September 21, 2017.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[INFO] graphlab.cython.cy_server: GraphLab Create v2.1 started. Logging: /tmp/graphlab_server_1480536700.log\n"
     ]
    }
   ],
   "source": [
    "import graphlab as gl\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done reading\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method SFrame.head of Columns:\n",
       "\tNo.\tint\n",
       "\tTime\tfloat\n",
       "\tSource\tstr\n",
       "\tDestination\tstr\n",
       "\tProtocol\tstr\n",
       "\tLength\tint\n",
       "\tInfo\tstr\n",
       "\tDestInation Port\tstr\n",
       "\tSource Port\tstr\n",
       "\tProtocols in frame\tstr\n",
       "\tDifferentiated Services Field\tstr\n",
       "\tIP_Flags\tstr\n",
       "\tudp_Length\tstr\n",
       "\tTCP Segment Len\tstr\n",
       "\tSequence number\tstr\n",
       "\ttcp_Flags\tstr\n",
       "\tAnswer RRs\tstr\n",
       "\tSource GeoIP Country\tstr\n",
       "\tSource GeoIP Latitude\tstr\n",
       "\tSource GeoIP Longitude\tstr\n",
       "\tDestination GeoIP Latitude\tstr\n",
       "\tDestination GeoIP Longitude\tstr\n",
       "\tDestination GeoIP Country\tstr\n",
       "\tTime to live\tstr\n",
       "\tNext sequence number\tstr\n",
       "\n",
       "Rows: 9388270\n",
       "\n",
       "Data:\n",
       "+-----+------------+-------------------+-------------+----------+--------+\n",
       "| No. |    Time    |       Source      | Destination | Protocol | Length |\n",
       "+-----+------------+-------------------+-------------+----------+--------+\n",
       "|  1  |    0.0     | CiscoInc_db:19:c3 |  Broadcast  |   ARP    |   60   |\n",
       "|  2  |  8.982709  | CiscoInc_db:19:c3 |  Broadcast  |   ARP    |   60   |\n",
       "|  3  | 50.099564  | CiscoInc_db:19:c3 |  Broadcast  |   ARP    |   60   |\n",
       "|  4  | 50.369266  | 54:52:00:00:00:01 |  Broadcast  |   ARP    |   60   |\n",
       "|  5  | 51.369054  | 54:52:00:00:00:01 |  Broadcast  |   ARP    |   60   |\n",
       "|  6  | 52.369688  | 54:52:00:00:00:01 |  Broadcast  |   ARP    |   60   |\n",
       "|  7  |  53.08684  | CiscoInc_db:19:c3 |  Broadcast  |   ARP    |   60   |\n",
       "|  8  | 59.086131  | CiscoInc_db:19:c3 |  Broadcast  |   ARP    |   60   |\n",
       "|  9  | 160.084662 | CadmusCo_b5:b7:19 |  Broadcast  |   ARP    |   60   |\n",
       "|  10 | 160.084668 | CadmusCo_b5:b7:19 |  Broadcast  |   ARP    |   60   |\n",
       "+-----+------------+-------------------+-------------+----------+--------+\n",
       "+-------------------------------+------------------+-------------+\n",
       "|              Info             | DestInation Port | Source Port |\n",
       "+-------------------------------+------------------+-------------+\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Who has 147.32.84.165? Tel... |                  |             |\n",
       "| Gratuitous ARP for 147.32.... |                  |             |\n",
       "| Gratuitous ARP for 147.32.... |                  |             |\n",
       "+-------------------------------+------------------+-------------+\n",
       "+--------------------+-------------------------------+----------+------------+-----------------+\n",
       "| Protocols in frame | Differentiated Services Field | IP_Flags | udp_Length | TCP Segment Len |\n",
       "+--------------------+-------------------------------+----------+------------+-----------------+\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "| eth:ethertype:arp  |                               |          |            |                 |\n",
       "+--------------------+-------------------------------+----------+------------+-----------------+\n",
       "+-----------------+-----------+------------+----------------------+-----------------------+\n",
       "| Sequence number | tcp_Flags | Answer RRs | Source GeoIP Country | Source GeoIP Latitude |\n",
       "+-----------------+-----------+------------+----------------------+-----------------------+\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "|                 |           |            |                      |                       |\n",
       "+-----------------+-----------+------------+----------------------+-----------------------+\n",
       "+------------------------+-----+\n",
       "| Source GeoIP Longitude | ... |\n",
       "+------------------------+-----+\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "|                        | ... |\n",
       "+------------------------+-----+\n",
       "[9388270 rows x 25 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF = gl.SFrame.read_csv('ISCX_Botnet-Training.csv',verbose=False)\n",
    "print \"Done reading\"\n",
    "SF.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "SF = SF[(SF['Source Port']!='')&(SF['DestInation Port']!='')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">No.</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Time</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Source</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Destination</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Protocol</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Length</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Info</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Destination Port</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">15</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">162.765245</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">110</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Registration NB<br>SARUMAN&lt;00&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">16</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">162.765253</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">110</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Registration NB<br>SARUMAN&lt;00&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">17</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.510681</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">110</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Registration NB<br>SARUMAN&lt;00&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">18</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.510692</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">110</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Registration NB<br>SARUMAN&lt;00&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">22</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.929961</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.80.9</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">DNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">64</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Standard query 0x6f4c A<br>wpad ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">23</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.929967</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.80.9</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">DNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">64</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Standard query 0x6f4c A<br>wpad ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">24</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.930586</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.80.9</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">DNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">139</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Standard query response<br>0x6f4c No such name A ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">25</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.930994</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">92</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Name query NB WPAD.&lt;00&gt;</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">26</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">163.930999</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">92</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Name query NB WPAD.&lt;00&gt;</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">27</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">164.262142</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.180</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">147.32.84.255</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">NBNS</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">110</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Registration NB<br>SARUMAN&lt;00&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "    </tr>\n",
       "</table>\n",
       "<table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Source Port</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Protocols in frame</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Differentiated Services<br>Field ...</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">IP_Flags</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">udp_Length</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">TCP Segment Len</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Sequence number</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">76</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">76</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">76</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">76</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1025</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:dns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">30</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1025</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:dns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">30</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">53</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:dns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">105</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">58</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">58</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">137</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">eth:ethertype:ip:udp:nbns</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0x00</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">76</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "</table>\n",
       "<table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">tcp_Flags</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Answer RRs</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Source GeoIP Country</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Source GeoIP Latitude</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Source GeoIP Longitude</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Destination GeoIP<br>Latitude ...</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">50.083302</td>\n",
       "    </tr>\n",
       "</table>\n",
       "<table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Destination GeoIP<br>Longitude ...</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Destination GeoIP Country</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Time to live</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">Next sequence number</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">62</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">14.4667</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Czech Republic</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">128</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\"></td>\n",
       "    </tr>\n",
       "</table>\n",
       "[? rows x 25 columns]<br/>Note: Only the head of the SFrame is printed. This SFrame is lazily evaluated.<br/>You can use sf.materialize() to force materialization.\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tNo.\tint\n",
       "\tTime\tfloat\n",
       "\tSource\tstr\n",
       "\tDestination\tstr\n",
       "\tProtocol\tstr\n",
       "\tLength\tint\n",
       "\tInfo\tstr\n",
       "\tDestination Port\tstr\n",
       "\tSource Port\tstr\n",
       "\tProtocols in frame\tstr\n",
       "\tDifferentiated Services Field\tstr\n",
       "\tIP_Flags\tstr\n",
       "\tudp_Length\tstr\n",
       "\tTCP Segment Len\tstr\n",
       "\tSequence number\tstr\n",
       "\ttcp_Flags\tstr\n",
       "\tAnswer RRs\tstr\n",
       "\tSource GeoIP Country\tstr\n",
       "\tSource GeoIP Latitude\tstr\n",
       "\tSource GeoIP Longitude\tstr\n",
       "\tDestination GeoIP Latitude\tstr\n",
       "\tDestination GeoIP Longitude\tstr\n",
       "\tDestination GeoIP Country\tstr\n",
       "\tTime to live\tstr\n",
       "\tNext sequence number\tstr\n",
       "\n",
       "Rows: Unknown\n",
       "\n",
       "Data:\n",
       "+-----+------------+---------------+---------------+----------+--------+\n",
       "| No. |    Time    |     Source    |  Destination  | Protocol | Length |\n",
       "+-----+------------+---------------+---------------+----------+--------+\n",
       "|  15 | 162.765245 | 147.32.84.180 | 147.32.84.255 |   NBNS   |  110   |\n",
       "|  16 | 162.765253 | 147.32.84.180 | 147.32.84.255 |   NBNS   |  110   |\n",
       "|  17 | 163.510681 | 147.32.84.180 | 147.32.84.255 |   NBNS   |  110   |\n",
       "|  18 | 163.510692 | 147.32.84.180 | 147.32.84.255 |   NBNS   |  110   |\n",
       "|  22 | 163.929961 | 147.32.84.180 |  147.32.80.9  |   DNS    |   64   |\n",
       "|  23 | 163.929967 | 147.32.84.180 |  147.32.80.9  |   DNS    |   64   |\n",
       "|  24 | 163.930586 |  147.32.80.9  | 147.32.84.180 |   DNS    |  139   |\n",
       "|  25 | 163.930994 | 147.32.84.180 | 147.32.84.255 |   NBNS   |   92   |\n",
       "|  26 | 163.930999 | 147.32.84.180 | 147.32.84.255 |   NBNS   |   92   |\n",
       "|  27 | 164.262142 | 147.32.84.180 | 147.32.84.255 |   NBNS   |  110   |\n",
       "+-----+------------+---------------+---------------+----------+--------+\n",
       "+-------------------------------+------------------+-------------+\n",
       "|              Info             | Destination Port | Source Port |\n",
       "+-------------------------------+------------------+-------------+\n",
       "|  Registration NB SARUMAN<00>  |       137        |     137     |\n",
       "|  Registration NB SARUMAN<00>  |       137        |     137     |\n",
       "|  Registration NB SARUMAN<00>  |       137        |     137     |\n",
       "|  Registration NB SARUMAN<00>  |       137        |     137     |\n",
       "|  Standard query 0x6f4c A wpad |        53        |     1025    |\n",
       "|  Standard query 0x6f4c A wpad |        53        |     1025    |\n",
       "| Standard query response 0x... |       1025       |      53     |\n",
       "|    Name query NB WPAD.<00>    |       137        |     137     |\n",
       "|    Name query NB WPAD.<00>    |       137        |     137     |\n",
       "|  Registration NB SARUMAN<00>  |       137        |     137     |\n",
       "+-------------------------------+------------------+-------------+\n",
       "+---------------------------+-------------------------------+----------+------------+\n",
       "|     Protocols in frame    | Differentiated Services Field | IP_Flags | udp_Length |\n",
       "+---------------------------+-------------------------------+----------+------------+\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     76     |\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     76     |\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     76     |\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     76     |\n",
       "|  eth:ethertype:ip:udp:dns |              0x00             |   0x00   |     30     |\n",
       "|  eth:ethertype:ip:udp:dns |              0x00             |   0x00   |     30     |\n",
       "|  eth:ethertype:ip:udp:dns |              0x00             |   0x00   |    105     |\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     58     |\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     58     |\n",
       "| eth:ethertype:ip:udp:nbns |              0x00             |   0x00   |     76     |\n",
       "+---------------------------+-------------------------------+----------+------------+\n",
       "+-----------------+-----------------+-----------+------------+----------------------+\n",
       "| TCP Segment Len | Sequence number | tcp_Flags | Answer RRs | Source GeoIP Country |\n",
       "+-----------------+-----------------+-----------+------------+----------------------+\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "|                 |                 |           |     0      |    Czech Republic    |\n",
       "|                 |                 |           |     0      |    Czech Republic    |\n",
       "|                 |                 |           |     0      |    Czech Republic    |\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "|                 |                 |           |            |    Czech Republic    |\n",
       "+-----------------+-----------------+-----------+------------+----------------------+\n",
       "+-----------------------+------------------------+-----+\n",
       "| Source GeoIP Latitude | Source GeoIP Longitude | ... |\n",
       "+-----------------------+------------------------+-----+\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "+-----------------------+------------------------+-----+\n",
       "[? rows x 25 columns]\n",
       "Note: Only the head of the SFrame is printed. This SFrame is lazily evaluated.\n",
       "You can use sf.materialize() to force materialization."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF.rename({'DestInation Port':'Destination Port'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "cdf125ff-1c85-4b35-904d-54aa6833cae0"
    }
   },
   "outputs": [],
   "source": [
    "SF['tcp_Flags'] = SF['tcp_Flags'].apply(lambda x:int(x,16) if x!='' else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "nbpresent": {
     "id": "aa29b2de-eb6d-4c22-a372-1678edf2680e"
    }
   },
   "outputs": [],
   "source": [
    "Protocols = sorted(SF['Protocol'].unique())\n",
    "#print Protocols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some analysis for number of packets of each protocol type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "IRC\n",
    "    \"192.168.2.112\",\n",
    "    \"131.202.243.84\",\n",
    "    \"192.168.5.122\",\n",
    "    \"198.164.30.2\",\n",
    "    \"192.168.2.110\",\n",
    "    \"192.168.5.122\",\n",
    "    \"192.168.4.118\",\n",
    "    \"192.168.5.122\",\n",
    "    \"192.168.2.113\",\n",
    "    \"192.168.5.122\",\n",
    "    \"192.168.1.103\",\n",
    "    \"192.168.5.122\",\n",
    "    \"192.168.4.120\",\n",
    "    \"192.168.5.122\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.2.110\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.4.120\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.1.103\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.2.113\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.4.118\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.2.109\",\n",
    "    \"192.168.2.112\",\n",
    "    \"192.168.2.105\",\n",
    "    \"192.168.1.105\",\n",
    "    \"192.168.5.122\","
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "iplist = [\n",
    "    \"147.32.84.180\",\n",
    "    \"147.32.84.170\",\n",
    "    \"147.32.84.150\",\n",
    "    \"147.32.84.140\",\n",
    "    \"147.32.84.130\",\n",
    "    \"147.32.84.160\",\n",
    "    \"10.0.2.15\",\n",
    "    \"192.168.106.141\",\n",
    "    \"192.168.106.131\",\n",
    "    \"172.16.253.130\",\n",
    "    \"172.16.253.131\",\n",
    "    \"172.16.253.129\",\n",
    "    \"172.16.253.240\",\n",
    "    \"74.78.117.238\",\n",
    "    \"158.65.110.24\",\n",
    "    \"192.168.3.35\",\n",
    "    \"192.168.3.25\",\n",
    "    \"192.168.3.65\",\n",
    "    \"172.29.0.116\",\n",
    "    \"172.29.0.109\",\n",
    "    \"172.16.253.132\",\n",
    "    \"192.168.248.165\",\n",
    "    \"10.37.130.4\",\n",
    "    \"131.202.243.84\", \n",
    "    \"198.164.30.2\", \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \"192.168.5.122\", \"192.168.5.122\", \n",
    "    \"192.168.2.110\", \n",
    "    \"192.168.4.120\", \"192.168.1.103\",\n",
    "    \"192.168.2.113\", \"192.168.4.118\" , \"192.168.2.109\", \"192.168.2.105\", \"192.168.5.122\"]\n",
    "\n",
    "MasterBot = [ (\"192.168.2.112\", \"131.202.243.84\"), (\"192.168.5.122\", \"198.164.30.2\"), (\"192.168.2.110\", \"192.168.5.122\"),( \"192.168.4.118\", \"192.168.5.122\"), (\"192.168.2.113\", \"192.168.5.122\"), (\"192.168.1.103\", \"192.168.5.122\"), (\"192.168.4.120\", \"192.168.5.122\"), (\"192.168.2.112\", \"192.168.2.110\"), (\"192.168.2.112\", \"192.168.4.120\"), (\"192.168.2.112\", \"192.168.1.103\"), (\"192.168.2.112\", \"192.168.2.113\"), (\"192.168.2.112\", \"192.168.4.118\"), (\"192.168.2.112\", \"192.168.2.109\"), (\"192.168.2.112\", \"192.168.2.105\"), (\"192.168.1.105\", \"192.168.5.122\")]\n",
    "\n",
    "iplist = gl.SArray(iplist)\n",
    "iplist = iplist.unique() \n",
    "iplist = list(iplist)\n",
    "\n",
    "def fcheckIP(x):\n",
    "    if (x['Source'] in iplist) or (x['Destination'] in iplist):\n",
    "        return 1\n",
    "    else:\n",
    "        if ((x['Source'],x['Destination'])  in MasterBot ) or ((x['Destination'],x['Source'])  in MasterBot ) :\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "        \n",
    "\n",
    "SF['isBot'] = SF[['Source','Destination']].apply(lambda x: fcheckIP(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Botnet Labelled packets and their percentage.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3874036 41.3869521207\n"
     ]
    }
   ],
   "source": [
    "temp = len(SF[SF['isBot']==1])\n",
    "print temp, (temp*1.0)/len(SF)*100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sorting will help in identifying flow effectively"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flow Identification Bidirectional"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No More Brainstorming, this is TRUE !!!!!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Sorting\n",
      "Running 0 Done !\n",
      "Running 500000 Done !\n",
      "Running 1000000 Done !\n",
      "Running 1500000 Done !\n",
      "Running 2000000 Done !\n",
      "Running 2500000 Done !\n",
      "Running 3000000 Done !\n",
      "Running 3500000 Done !\n",
      "Running 4000000 Done !\n",
      "Running 4500000 Done !\n",
      "Running 5000000 Done !\n",
      "Running 5500000 Done !\n",
      "Running 6000000 Done !\n",
      "Running 6500000 Done !\n",
      "Running 7000000 Done !\n",
      "Running 7500000 Done !\n",
      "Running 8000000 Done !\n",
      "Running 8500000 Done !\n",
      "Running 9000000 Done !\n",
      "504043\n",
      "404434\n"
     ]
    }
   ],
   "source": [
    "def flow_id(x):\n",
    "    if x['Source']>x['Destination']:\n",
    "        return x['Source']+'-'+x['Destination']+'-'+str(x['Source Port'])+'-'+str(x['Destination Port'])+'-'+x['Protocol']\n",
    "    else:\n",
    "        return x['Destination']+'-'+x['Source']+'-'+str(x['Destination Port'])+'-'+str(x['Source Port'])+'-'+x['Protocol']\n",
    "SF['UFid'] = SF.apply(lambda x:flow_id(x))\n",
    "\n",
    "#For identifying IOPR\n",
    "SF['Forward'] = SF.apply(lambda x: 1 if x['Source']>x['Destination'] else 0 )\n",
    "\n",
    "## function for comparing two different flows based on columns\n",
    "def compareUF(x,y):\n",
    "    if x!=y:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "\n",
    "\n",
    "##Code for logic of Bidirectional flow identification\n",
    "import pickle \n",
    "\n",
    "FlowNo = 0 ##Unique Flow Number for each flow\n",
    "prev = None\n",
    "Flow = []     ##Stores all flows in form of list of dictionary \n",
    "#cFlow = []    ##Store the current flow (all details)\n",
    "count = 0\n",
    "fc = 0\n",
    "startTime = None   ##Start Time of each flow to implement timeout\n",
    "SF = SF.sort(['UFid','Time'])\n",
    "print 'Done Sorting'\n",
    "for x in SF:\n",
    "    if count%500000 == 0:\n",
    "        print 'Running '+str(count)+' Done !'\n",
    "        \n",
    "    count = count+1\n",
    "    \n",
    "    if prev is None:\n",
    "        if startTime is None:\n",
    "            startTime = x['Time']\n",
    "        Flow.append(FlowNo)\n",
    "        \n",
    "        prev = x['UFid']\n",
    "    elif compareUF(x['UFid'],prev):\n",
    "        if x['tcp_Flags']&1:\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = None\n",
    "            FlowNo = FlowNo + 1\n",
    "\n",
    "        elif x['Time']-startTime>=3600:\n",
    "            FlowNo = FlowNo + 1\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = x['Time']\n",
    "            \n",
    "        else:\n",
    "            \n",
    "            Flow.append(FlowNo)\n",
    "            prev = x['UFid']\n",
    "\n",
    "    else:\n",
    "        FlowNo = FlowNo + 1\n",
    "        Flow.append(FlowNo)\n",
    "        prev = x['UFid']\n",
    "        startTime = x['Time']\n",
    "\n",
    "\n",
    "print len(gl.SArray(Flow).unique())\n",
    "\n",
    "\n",
    "\n",
    "SF['Flow'] = gl.SArray(Flow)\n",
    "temp = SF.groupby('Flow',{\n",
    "            'Count':gl.aggregate.COUNT()\n",
    "        })\n",
    "print len(temp[temp['Count']>1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Sorting\n",
      "Running 0 Done !\n",
      "Running 500000 Done !\n",
      "Running 1000000 Done !\n",
      "Running 1500000 Done !\n",
      "Running 2000000 Done !\n",
      "Running 2500000 Done !\n",
      "Running 3000000 Done !\n",
      "Running 3500000 Done !\n",
      "Running 4000000 Done !\n",
      "Running 4500000 Done !\n",
      "Running 5000000 Done !\n",
      "Running 5500000 Done !\n",
      "Running 6000000 Done !\n",
      "Running 6500000 Done !\n",
      "Running 7000000 Done !\n",
      "Running 7500000 Done !\n",
      "Running 8000000 Done !\n",
      "Running 8500000 Done !\n",
      "Running 9000000 Done !\n",
      "519392\n",
      "420390\n"
     ]
    }
   ],
   "source": [
    "\n",
    "##Code for logic of Bidirectional flow identification\n",
    "import pickle \n",
    "\n",
    "FlowNo = 0 ##Unique Flow Number for each flow\n",
    "prev = None\n",
    "Flow = []     ##Stores all flows in form of list of dictionary \n",
    "#cFlow = []    ##Store the current flow (all details)\n",
    "count = 0\n",
    "fc = 0\n",
    "startTime = None   ##Start Time of each flow to implement timeout\n",
    "#SF = SF.sort(['UFid','Time'])\n",
    "print 'Done Sorting'\n",
    "for x in SF:\n",
    "    if count%500000 == 0:\n",
    "        print 'Running '+str(count)+' Done !'\n",
    "        \n",
    "    count = count+1\n",
    "    \n",
    "    if prev is None:\n",
    "        if startTime is None:\n",
    "            startTime = x['Time']\n",
    "        Flow.append(FlowNo)\n",
    "        \n",
    "        prev = x['UFid']\n",
    "    elif compareUF(x['UFid'],prev):\n",
    "        if x['tcp_Flags']&1 or x['tcp_Flags']&4:\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = None\n",
    "            FlowNo = FlowNo + 1\n",
    "\n",
    "        elif x['Time']-startTime>=3600:\n",
    "            FlowNo = FlowNo + 1\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = x['Time']\n",
    "            \n",
    "        else:\n",
    "            \n",
    "            Flow.append(FlowNo)\n",
    "            prev = x['UFid']\n",
    "\n",
    "    else:\n",
    "        FlowNo = FlowNo + 1\n",
    "        Flow.append(FlowNo)\n",
    "        prev = x['UFid']\n",
    "        startTime = x['Time']\n",
    "\n",
    "\n",
    "print len(gl.SArray(Flow).unique())\n",
    "\n",
    "\n",
    "\n",
    "SF['Flow'] = gl.SArray(Flow)\n",
    "temp = SF.groupby('Flow',{\n",
    "            'Count':gl.aggregate.COUNT()\n",
    "        })\n",
    "print len(temp[temp['Count']>1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def flow_id(x):\n",
    "    if x['Source']>x['Destination']:\n",
    "        return x['Source']+'-'+x['Destination']+'-'+str(x['Source Port'])+'-'+str(x['Destination Port'])+'-'+x['Protocol']\n",
    "    else:\n",
    "        return x['Destination']+'-'+x['Source']+'-'+str(x['Destination Port'])+'-'+str(x['Source Port'])+'-'+x['Protocol']\n",
    "SF['UFid'] = SF.apply(lambda x:flow_id(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#For identifying IOPR\n",
    "SF['Forward'] = SF.apply(lambda x: 1 if x['Source']>x['Destination'] else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## function for comparing two different flows based on columns\n",
    "def compareUF(x,y):\n",
    "    if x!=y:\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Sorting\n",
      "Running 0 Done !\n",
      "Running 500000 Done !\n",
      "Running 1000000 Done !\n",
      "Running 1500000 Done !\n",
      "Running 2000000 Done !\n",
      "Running 2500000 Done !\n",
      "Running 3000000 Done !\n",
      "Running 3500000 Done !\n",
      "Running 4000000 Done !\n",
      "Running 4500000 Done !\n",
      "Running 5000000 Done !\n",
      "Running 5500000 Done !\n",
      "Running 6000000 Done !\n",
      "Running 6500000 Done !\n",
      "Running 7000000 Done !\n",
      "Running 7500000 Done !\n",
      "Running 8000000 Done !\n",
      "Running 8500000 Done !\n",
      "Running 9000000 Done !\n",
      "504044\n"
     ]
    }
   ],
   "source": [
    "##Code for logic of Bidirectional flow identification\n",
    "import pickle \n",
    "\n",
    "FlowNo = 0 ##Unique Flow Number for each flow\n",
    "prev = None\n",
    "Flow = []     ##Stores all flows in form of list of dictionary \n",
    "#cFlow = []    ##Store the current flow (all details)\n",
    "count = 0\n",
    "fc = 0\n",
    "startTime = None   ##Start Time of each flow to implement timeout\n",
    "SF = SF.sort(['UFid','Time'])\n",
    "print 'Done Sorting'\n",
    "for x in SF:\n",
    "    if count%500000 == 0:\n",
    "        print 'Running '+str(count)+' Done !'\n",
    "        \n",
    "    count = count+1\n",
    "    \n",
    "    if prev is None:\n",
    "        if startTime is None:\n",
    "            startTime = x['Time']\n",
    "        Flow.append(FlowNo)\n",
    "        \n",
    "        prev = x['UFid']\n",
    "    elif compareUF(x['UFid'],prev):\n",
    "        if x['tcp_Flags']&1:\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = None\n",
    "            FlowNo = FlowNo + 1\n",
    "\n",
    "        elif x['Time']-startTime>=3600:\n",
    "            FlowNo = FlowNo + 1\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = x['Time']\n",
    "            \n",
    "        else:\n",
    "            \n",
    "            Flow.append(FlowNo)\n",
    "            prev = x['UFid']\n",
    "\n",
    "    else:\n",
    "        FlowNo = FlowNo + 1\n",
    "        Flow.append(FlowNo)\n",
    "        prev = x['UFid']\n",
    "        startTime = x['Time']\n",
    "\n",
    "\n",
    "print len(gl.SArray(Flow).unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "404434"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF['Flow'] = gl.SArray(Flow)\n",
    "temp = SF.groupby('Flow',{\n",
    "            'Count':gl.aggregate.COUNT()\n",
    "        })\n",
    "len(temp[temp['Count']>1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "238501 0.473174960916\n"
     ]
    }
   ],
   "source": [
    "temp = SF.groupby('Flow',{\n",
    "        'NumBots' : gl.aggregate.SUM('isBot')\n",
    "    })\n",
    "NumBotFlows = len(temp[temp['NumBots']>1])\n",
    "print NumBotFlows, NumBotFlows*1.0/len(SF['Flow'].unique()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9360525"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Flow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9360525"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "SF['FlowNo.'] = gl.SArray(Flow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##Code for checking authenticity of flow logic\n",
    "#STD[(STD['Source']=='0.0.0.0')&(STD['Destination']=='255.255.255.255')&(STD['Source Port']=='68')&(STD['Destination Port']=='67')].sort('Time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'count = 0\\nfor li in Flow:\\n    for i in range(1,len(li)):\\n        if li[i]<li[i-1]:\\n            #print li\\n            count = count+1\\n            break;\\nprint count'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Code to check if in any flows there are some No.s which are in decreasing order (Indicative or Decrepancies)\n",
    "## UPDATE: No. does not indicate same relation in time, so Data collected is right !\n",
    "\"\"\"count = 0\n",
    "for li in Flow:\n",
    "    for i in range(1,len(li)):\n",
    "        if li[i]<li[i-1]:\n",
    "            #print li\n",
    "            count = count+1\n",
    "            break;\n",
    "print count\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(Flow,open('Flow.pkl','w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "SF.save('ISCX_Botnet-Testing_Ports_Only_Sorted_Flow_BD.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tfn(x):\n",
    "    if 'udp' in x.split(':'):\n",
    "        return 1\n",
    "    return 0\n",
    "SF['hasUDP'] = SF['Protocols in frame'].apply(lambda x:tfn(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method SFrame.head of Columns:\n",
       "\tNo.\tint\n",
       "\tTime\tfloat\n",
       "\tSource\tstr\n",
       "\tDestination\tstr\n",
       "\tProtocol\tstr\n",
       "\tLength\tint\n",
       "\tInfo\tstr\n",
       "\tDestination Port\tstr\n",
       "\tSource Port\tstr\n",
       "\tProtocols in frame\tstr\n",
       "\tDifferentiated Services Field\tstr\n",
       "\tIP_Flags\tstr\n",
       "\tudp_Length\tstr\n",
       "\tTCP Segment Len\tstr\n",
       "\tSequence number\tstr\n",
       "\ttcp_Flags\tint\n",
       "\tAnswer RRs\tstr\n",
       "\tSource GeoIP Country\tstr\n",
       "\tSource GeoIP Latitude\tstr\n",
       "\tSource GeoIP Longitude\tstr\n",
       "\tDestination GeoIP Latitude\tstr\n",
       "\tDestination GeoIP Longitude\tstr\n",
       "\tDestination GeoIP Country\tstr\n",
       "\tTime to live\tstr\n",
       "\tNext sequence number\tstr\n",
       "\tisBot\tint\n",
       "\tUFid\tstr\n",
       "\tForward\tint\n",
       "\tFlow\tint\n",
       "\tFlowNo.\tint\n",
       "\thasUDP\tint\n",
       "\n",
       "Rows: 9360525\n",
       "\n",
       "Data:\n",
       "+--------+---------------+---------------+-----------------+----------+--------+\n",
       "|  No.   |      Time     |     Source    |   Destination   | Protocol | Length |\n",
       "+--------+---------------+---------------+-----------------+----------+--------+\n",
       "| 819351 | 453240.146507 | 147.32.84.160 |   1.22.16.143   |   TCP    |   62   |\n",
       "| 819352 | 453240.146514 | 147.32.84.160 |   1.22.16.143   |   TCP    |   62   |\n",
       "| 819384 | 453243.150744 | 147.32.84.160 |   1.22.16.143   |   TCP    |   62   |\n",
       "| 819385 | 453243.150747 | 147.32.84.160 |   1.22.16.143   |   TCP    |   62   |\n",
       "| 819616 | 453249.159548 | 147.32.84.160 |   1.22.16.143   |   TCP    |   62   |\n",
       "| 819617 | 453249.159556 | 147.32.84.160 |   1.22.16.143   |   TCP    |   62   |\n",
       "| 819151 | 453192.242578 | 147.32.84.160 | 109.200.239.148 |   TCP    |   62   |\n",
       "| 819152 | 453192.242585 | 147.32.84.160 | 109.200.239.148 |   TCP    |   62   |\n",
       "| 819178 | 453195.182192 | 147.32.84.160 | 109.200.239.148 |   TCP    |   62   |\n",
       "| 819179 | 453195.182195 | 147.32.84.160 | 109.200.239.148 |   TCP    |   62   |\n",
       "+--------+---------------+---------------+-----------------+----------+--------+\n",
       "+-------------------------------+------------------+-------------+\n",
       "|              Info             | Destination Port | Source Port |\n",
       "+-------------------------------+------------------+-------------+\n",
       "| 1085  >  3128 [SYN] Seq=0 ... |       3128       |     1085    |\n",
       "| [TCP Out-Of-Order] 1085  >... |       3128       |     1085    |\n",
       "| [TCP Retransmission] 1085 ... |       3128       |     1085    |\n",
       "| [TCP Retransmission] 1085 ... |       3128       |     1085    |\n",
       "| [TCP Retransmission] 1085 ... |       3128       |     1085    |\n",
       "| [TCP Retransmission] 1085 ... |       3128       |     1085    |\n",
       "| 1066  >  3128 [SYN] Seq=0 ... |       3128       |     1066    |\n",
       "| [TCP Out-Of-Order] 1066  >... |       3128       |     1066    |\n",
       "| [TCP Retransmission] 1066 ... |       3128       |     1066    |\n",
       "| [TCP Retransmission] 1066 ... |       3128       |     1066    |\n",
       "+-------------------------------+------------------+-------------+\n",
       "+----------------------+-------------------------------+----------+------------+\n",
       "|  Protocols in frame  | Differentiated Services Field | IP_Flags | udp_Length |\n",
       "+----------------------+-------------------------------+----------+------------+\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "| eth:ethertype:ip:tcp |              0x00             |   0x02   |            |\n",
       "+----------------------+-------------------------------+----------+------------+\n",
       "+-----------------+-----------------+-----------+------------+----------------------+\n",
       "| TCP Segment Len | Sequence number | tcp_Flags | Answer RRs | Source GeoIP Country |\n",
       "+-----------------+-----------------+-----------+------------+----------------------+\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "|        0        |        0        |     2     |            |    Czech Republic    |\n",
       "+-----------------+-----------------+-----------+------------+----------------------+\n",
       "+-----------------------+------------------------+-----+\n",
       "| Source GeoIP Latitude | Source GeoIP Longitude | ... |\n",
       "+-----------------------+------------------------+-----+\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "|       50.083302       |        14.4667         | ... |\n",
       "+-----------------------+------------------------+-----+\n",
       "[9360525 rows x 31 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF.head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Ratio of incoming to outgoing packets\n",
    "temp = SF.groupby('FlowNo.',{\n",
    "        'NumForward' : gl.aggregate.SUM('Forward'),\n",
    "        'Total' : gl.aggregate.COUNT()\n",
    "    })\n",
    "temp['IOPR']= temp.apply(lambda x: ((x['Total']-x['NumForward'])*1.0)/x['NumForward'] if x['NumForward'] !=0 else (-1) )\n",
    "temp = temp['FlowNo.','IOPR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "348937"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(temp[temp['IOPR']!=-1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "SF = SF.join(temp,on='FlowNo.')\n",
    "SF.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype: float\n",
       "Rows: 9360525\n",
       "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.0, 0.0, 0.0, 0.0, -1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ... ]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF['IOPR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "504044\n",
      "504044\n",
      "+---------+------------+\n",
      "| FlowNo. | NumPackets |\n",
      "+---------+------------+\n",
      "|  211023 |     1      |\n",
      "|  442699 |     2      |\n",
      "|  79732  |     4      |\n",
      "+---------+------------+\n",
      "[3 rows x 2 columns]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Answer RRs',\n",
       " 'BytesEx',\n",
       " 'Destination',\n",
       " 'Destination GeoIP Country',\n",
       " 'Destination GeoIP Latitude',\n",
       " 'Destination GeoIP Longitude',\n",
       " 'Destination Port',\n",
       " 'Differentiated Services Field',\n",
       " 'Duration',\n",
       " 'FPL',\n",
       " 'Flow',\n",
       " 'FlowNo.',\n",
       " 'Forward',\n",
       " 'IOPR',\n",
       " 'IP_Flags',\n",
       " 'Info',\n",
       " 'Length',\n",
       " 'Next sequence number',\n",
       " 'No.',\n",
       " 'NumPackets',\n",
       " 'Protocol',\n",
       " 'Protocols in frame',\n",
       " 'SameLenPktRatio',\n",
       " 'Sequence number',\n",
       " 'Source',\n",
       " 'Source GeoIP Country',\n",
       " 'Source GeoIP Latitude',\n",
       " 'Source GeoIP Longitude',\n",
       " 'Source Port',\n",
       " 'StdDevLen',\n",
       " 'TCP Segment Len',\n",
       " 'Time',\n",
       " 'Time to live',\n",
       " 'UFid',\n",
       " 'hasUDP',\n",
       " 'isBot',\n",
       " 'tcp_Flags',\n",
       " 'udp_Length']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## First Packet Length\n",
    "FlowFeatures = ['Source','Destination','Source Port','Destination Port','Protocol']\n",
    "FPL = SF.groupby(['FlowNo.'],{\n",
    "        'Time':gl.aggregate.MIN('Time')\n",
    "    })\n",
    "print len(FPL)\n",
    "FPL = FPL.join(SF,on =['FlowNo.','Time'])[['FlowNo.','Length']].unique()\n",
    "FPL = FPL.groupby(['FlowNo.'],{\n",
    "        'FPL':gl.aggregate.AVG('Length')\n",
    "    })\n",
    "print len(FPL)\n",
    "\n",
    "\n",
    "# In[33]:\n",
    "\n",
    "SF = SF.join(FPL, on ='FlowNo.')\n",
    "del(FPL)\n",
    "\n",
    "\n",
    "# ## 18/10/2016\n",
    "\n",
    "# In[35]:\n",
    "\n",
    "## Number of packets per flow\n",
    "temp = SF.groupby(['FlowNo.'],{\n",
    "        'NumPackets':gl.aggregate.COUNT()\n",
    "    })\n",
    "print temp.head(3)\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "del(temp)\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "## Number of bytes exchanged\n",
    "temp = SF.groupby(['FlowNo.'],{\n",
    "        'BytesEx':gl.aggregate.SUM('Length')\n",
    "    })\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "del(temp)\n",
    "\n",
    "\n",
    "# In[38]:\n",
    "\n",
    "## Standard deviation of packet length\n",
    "temp = SF.groupby(['FlowNo.'],{\n",
    "        'StdDevLen':gl.aggregate.STDV('Length')\n",
    "    })\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "del(temp)\n",
    "\n",
    "\n",
    "# In[40]:\n",
    "\n",
    "## Same length packet ratio\n",
    "temp2 = SF.groupby(['FlowNo.'],{\n",
    "        'SameLenPktRatio':gl.aggregate.COUNT_DISTINCT('Length')\n",
    "    })\n",
    "##temp from number of packets computation\n",
    "temp = SF.groupby(['FlowNo.'],{\n",
    "        'NumPackets':gl.aggregate.COUNT()\n",
    "    })\n",
    "temp = temp.join(temp2,on='FlowNo.')\n",
    "temp['SameLenPktRatio'] = temp['SameLenPktRatio']*1.0/temp['NumPackets']\n",
    "temp2 = None\n",
    "temp = temp[['FlowNo.','SameLenPktRatio']]\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[41]:\n",
    "\n",
    "## Duration of flow\n",
    "timeF = SF.groupby(['FlowNo.'],{\n",
    "        'startTime':gl.aggregate.MIN('Time'),\n",
    "        'endTime':gl.aggregate.MAX('Time')\n",
    "    })\n",
    "timeF['Duration'] = timeF['endTime'] - timeF['startTime']\n",
    "timeF = timeF[['FlowNo.','Duration']]\n",
    "SF = SF.join(timeF, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[45]:\n",
    "\n",
    "sorted(SF.column_names())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 5398 0\n",
      "0\n",
      "-121117184.726 780231.692992\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# In[50]:\n",
    "\n",
    "features = ['Answer RRs',\n",
    " 'BytesEx',\n",
    " 'Destination',\n",
    " 'Destination Port',\n",
    " 'Differentiated Services Field',\n",
    " 'Duration',\n",
    " 'FPL',\n",
    " 'IP_Flags',\n",
    " 'Info',\n",
    " 'Length',\n",
    " 'Next sequence number',\n",
    " 'No.',\n",
    " 'NumPackets',\n",
    " 'Protocol',\n",
    " 'Protocols in frame',\n",
    " 'SameLenPktRatio',\n",
    " 'Sequence number',\n",
    " 'Source',\n",
    " 'Source Port',\n",
    " 'StdDevLen',\n",
    " 'TCP Segment Len',\n",
    " 'Time',\n",
    " 'Time to live',\n",
    " 'isBot',\n",
    " 'tcp_Flags',\n",
    " 'FlowNo.',\n",
    " 'udp_Length',\n",
    " 'IOPR']\n",
    "SF = SF[features]\n",
    "\n",
    "\n",
    "# In[52]:\n",
    "\n",
    "## Average packets per second\n",
    "temp =  SF.groupby(['FlowNo.'],{\n",
    "        'NumPackets':gl.aggregate.COUNT()\n",
    "    })\n",
    "temp = temp.join(timeF,on=['FlowNo.'])\n",
    "temp['AvgPktPerSec'] = temp.apply(lambda x:0.0 if x['Duration'] == 0.0 else x['NumPackets']*1.0/x['Duration'])\n",
    "temp = temp[['FlowNo.','AvgPktPerSec']]\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[53]:\n",
    "\n",
    "##Average Bits Per Second\n",
    "temp = SF.groupby(['FlowNo.'],{\n",
    "        'BytesEx':gl.aggregate.SUM('Length')\n",
    "    })\n",
    "temp = temp.join(timeF,on=['FlowNo.'])\n",
    "temp['BitsPerSec'] = temp.apply(lambda x:0.0 if x['Duration'] == 0.0 else x['BytesEx']*8.0/x['Duration'])\n",
    "temp = temp[['FlowNo.','BitsPerSec']]\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[55]:\n",
    "\n",
    "## Average Packet Lentgth\n",
    "temp = SF.groupby(['FlowNo.'],{\n",
    "        'APL':gl.aggregate.AVG('Length')\n",
    "    })\n",
    "SF = SF.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "## Number of Reconnects, sort FlowNo, SeqNo\n",
    "\n",
    "\n",
    "# In[56]:\n",
    "\n",
    "def tfn(x):\n",
    "    if 'udp' in x.split(':') or 'tcp' in x.split(':'):\n",
    "        return 1\n",
    "    return 0\n",
    "temp = list(SF['Protocols in frame'].apply(lambda x:tfn(x)))\n",
    "\n",
    "\n",
    "# In[57]:\n",
    "\n",
    "len(temp)\n",
    "\n",
    "\n",
    "# In[58]:\n",
    "\n",
    "sum(temp)\n",
    "\n",
    "\n",
    "# In[60]:\n",
    "\n",
    "\n",
    "\n",
    "# In[62]:\n",
    "\n",
    "print len(SF[SF['Protocols in frame']=='eth:ethertype:ip:icmp:ip:tcp:http:urlencoded-form']),len(SF[SF['Protocols in frame']=='eth:ethertype:ip:icmp:ip:tcp']),len(SF[SF['Protocols in frame']=='eth:ethertype:ip:icmp:ip:tcp:http:data'])\n",
    "\n",
    "\n",
    "# In[64]:\n",
    "\n",
    "## Inter arrival time\n",
    "SF['IAT'] = 0\n",
    "SF = SF.sort(['FlowNo.','Time'])\n",
    "prev = None\n",
    "prevT = None\n",
    "li = []\n",
    "for x in SF:\n",
    "    if prev is None or x['FlowNo.']!= prev:\n",
    "        li.append(0)\n",
    "    else:\n",
    "        li.append(x['Time']-prevT)        \n",
    "    prev = x['FlowNo.']\n",
    "    prevT = x['Time']\n",
    "SF['IAT'] = gl.SArray(li)\n",
    "\n",
    "\n",
    "# In[65]:\n",
    "\n",
    "len(SF)\n",
    "\n",
    "\n",
    "# In[66]:\n",
    "\n",
    "print len(SF[SF['Protocols in frame']=='eth:ipx'])\n",
    "\n",
    "\n",
    "# In[67]:\n",
    "\n",
    "SF.save('Bidirectional_Train_Bot_features_till_IAT.csv')\n",
    "\n",
    "\n",
    "# # Is Null feature\n",
    "\n",
    "# ### Number of TCP Null packets\n",
    "\n",
    "# In[68]:\n",
    "\n",
    "\n",
    "# In[73]:\n",
    "\n",
    "## Null Packets handling\n",
    "def checkNull(x):\n",
    "    if(x['TCP Segment Len']=='0' or x['udp_Length']==8 ):\n",
    "        return 1\n",
    "    elif('ipx' in x['Protocols in frame'].split(':')):\n",
    "        l = x['Length'] - 30\n",
    "        if('eth' in x['Protocols in frame'].split(':')):\n",
    "            l = l - 14\n",
    "        if('ethtype' in x['Protocols in frame'].split(':')):\n",
    "            l = l - 2\n",
    "        if('llc' in x['Protocols in frame'].split(':')):\n",
    "            l = l - 8\n",
    "        if(l==0 or l==-1):\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "\n",
    "# In[74]:\n",
    "\n",
    "SF['isNull'] = SF.apply(lambda x:checkNull(x))\n",
    "\n",
    "\n",
    "# In[75]:\n",
    "\n",
    "\n",
    "\n",
    "# In[76]:\n",
    "\n",
    "NPEx = SF.groupby(['FlowNo.'],{\n",
    "        'NPEx':gl.aggregate.SUM('isNull')\n",
    "    })\n",
    "SF = SF.join(NPEx, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# ### Number of Reconnects - considering only TCP reconnects, using sequence number\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "\n",
    "# In[79]:\n",
    "\n",
    "recon = SF[SF['Sequence number']!=''].groupby(['FlowNo.'],{\n",
    "        'total_seq_no.' : gl.aggregate.COUNT('Sequence number'),\n",
    "        'distinct_seq_no.' : gl.aggregate.COUNT_DISTINCT('Sequence number')\n",
    "    })\n",
    "recon['reconnects'] = recon['total_seq_no.'] - recon['distinct_seq_no.']\n",
    "recon.head()\n",
    "recon = recon[['FlowNo.','reconnects']]\n",
    "SF = SF.join(recon,on='FlowNo.',how='left')\n",
    "len(SF)\n",
    "\n",
    "\n",
    "# In[81]:\n",
    "\n",
    "#To identify records where reconnect check was not applied like UDP etc\n",
    "SF.fillna('reconnects',-1)\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "#SF[SF['FlowNo.']==79732]['Info']\n",
    "\n",
    "\n",
    "# A lot of these reconnects can be simple retransmissions - due to out of order/timeout etcb\n",
    "\n",
    "# ### 22-10-2016\n",
    "\n",
    "# In[83]:\n",
    "\n",
    "\n",
    "\n",
    "# In[15]:\n",
    "\n",
    "print min(SF['Time']),max(SF['Time'])\n",
    "\n",
    "\n",
    "# # 28/10/16\n",
    "\n",
    "# Bidirectional Flowscombining all features and labels\n",
    "\n",
    "# In[84]:\n",
    "\n",
    "\n",
    "# In[85]:\n",
    "\n",
    "SF.save('Bidirectional_Train_Botnet_all_features.csv')\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "#sf_valid_train, sf_test = SF.random_split(.8, seed=5)\n",
    "#sf_valid, sf_train = sf_valid_train.random_split(.2, seed=5)\n",
    "#X_train, y_train = sf_train.drop('isBot'), sf_train['isBot']\n",
    "#X_valid, y_valid = sf_valid_train\n",
    "#X_train_valid, y_train_valid = X[:3475000], y[:3475000]\n",
    "#X_test, y_test = X[3475000:], y[3475000:]\n",
    "\n",
    "\n",
    "\n",
    "# In[89]:\n",
    "\n",
    "features = SF.column_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done reading\n",
      "Done Sorting\n",
      "Running 0 Done !\n",
      "Running 500000 Done !\n",
      "Running 1000000 Done !\n",
      "Running 1500000 Done !\n",
      "Running 2000000 Done !\n",
      "Running 2500000 Done !\n",
      "Running 3000000 Done !\n",
      "Running 3500000 Done !\n",
      "Running 4000000 Done !\n",
      "Running 4500000 Done !\n",
      "355434\n",
      "216964 0.610419937316\n",
      "355434\n",
      "355434\n",
      "+---------+------------+\n",
      "| FlowNo. | NumPackets |\n",
      "+---------+------------+\n",
      "|  211023 |     6      |\n",
      "|  79732  |     1      |\n",
      "|  333842 |     7      |\n",
      "+---------+------------+\n",
      "[3 rows x 2 columns]\n",
      "\n",
      "1 234 1\n",
      "5637\n",
      "6.985427 1359941622.99\n"
     ]
    }
   ],
   "source": [
    "SF2 = gl.SFrame.read_csv('ISCX_Botnet-Testing.csv',verbose=False)\n",
    "print \"Done reading\"\n",
    "SF2.head\n",
    "SF2 = SF2[(SF2['Source Port']!='')&(SF2['Destination Port']!='')]\n",
    "SF2['tcp_Flags'] = SF2['tcp_Flags'].apply(lambda x:int(x,16) if x!='' else 0)\n",
    "iplist = [\n",
    "    \"147.32.84.180\",\n",
    "    \"147.32.84.170\",\n",
    "    \"147.32.84.150\",\n",
    "    \"147.32.84.140\",\n",
    "    \"147.32.84.130\",\n",
    "    \"147.32.84.160\",\n",
    "    \"10.0.2.15\",\n",
    "    \"192.168.106.141\",\n",
    "    \"192.168.106.131\",\n",
    "    \"172.16.253.130\",\n",
    "    \"172.16.253.131\",\n",
    "    \"172.16.253.129\",\n",
    "    \"172.16.253.240\",\n",
    "    \"74.78.117.238\",\n",
    "    \"158.65.110.24\",\n",
    "    \"192.168.3.35\",\n",
    "    \"192.168.3.25\",\n",
    "    \"192.168.3.65\",\n",
    "    \"172.29.0.116\",\n",
    "    \"172.29.0.109\",\n",
    "    \"172.16.253.132\",\n",
    "    \"192.168.248.165\",\n",
    "    \"10.37.130.4\",\n",
    "    \"131.202.243.84\", \n",
    "    \"198.164.30.2\", \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \"192.168.5.122\", \"192.168.5.122\", \n",
    "    \"192.168.2.110\", \n",
    "    \"192.168.4.120\", \"192.168.1.103\",\n",
    "    \"192.168.2.113\", \"192.168.4.118\" , \"192.168.2.109\", \"192.168.2.105\", \"192.168.5.122\"]\n",
    "\n",
    "MasterBot = [ (\"192.168.2.112\", \"131.202.243.84\"), (\"192.168.5.122\", \"198.164.30.2\"), (\"192.168.2.110\", \"192.168.5.122\"),( \"192.168.4.118\", \"192.168.5.122\"), (\"192.168.2.113\", \"192.168.5.122\"), (\"192.168.1.103\", \"192.168.5.122\"), (\"192.168.4.120\", \"192.168.5.122\"), (\"192.168.2.112\", \"192.168.2.110\"), (\"192.168.2.112\", \"192.168.4.120\"), (\"192.168.2.112\", \"192.168.1.103\"), (\"192.168.2.112\", \"192.168.2.113\"), (\"192.168.2.112\", \"192.168.4.118\"), (\"192.168.2.112\", \"192.168.2.109\"), (\"192.168.2.112\", \"192.168.2.105\"), (\"192.168.1.105\", \"192.168.5.122\")]\n",
    "\n",
    "iplist = gl.SArray(iplist)\n",
    "iplist = iplist.unique() \n",
    "iplist = list(iplist)\n",
    "\n",
    "def fcheckIP(x):\n",
    "    if (x['Source'] in iplist) or (x['Destination'] in iplist):\n",
    "        return 1\n",
    "    else:\n",
    "        if ((x['Source'],x['Destination'])  in MasterBot ) or ((x['Destination'],x['Source'])  in MasterBot ) :\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "        \n",
    "\n",
    "SF2['isBot'] = SF2[['Source','Destination']].apply(lambda x: fcheckIP(x))\n",
    "\n",
    "def flow_id(x):\n",
    "    if x['Source']>x['Destination']:\n",
    "        return x['Source']+'-'+x['Destination']+'-'+str(x['Source Port'])+'-'+str(x['Destination Port'])+'-'+x['Protocol']\n",
    "    else:\n",
    "        return x['Destination']+'-'+x['Source']+'-'+str(x['Destination Port'])+'-'+str(x['Source Port'])+'-'+x['Protocol']\n",
    "SF2['UFid'] = SF2.apply(lambda x:flow_id(x))\n",
    "\n",
    "\n",
    "#For identifying IOPR\n",
    "SF2['Forward'] = SF2.apply(lambda x: 1 if x['Source']>x['Destination'] else 0 )\n",
    "\n",
    "\n",
    "## function for comparing two different flows based on columns\n",
    "def compareUF(x,y):\n",
    "    if x!=y:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "\n",
    "##Code for logic of Bidirectional flow identification\n",
    "import pickle \n",
    "\n",
    "FlowNo = 0 ##Unique Flow Number for each flow\n",
    "prev = None\n",
    "Flow = []     ##Stores all flows in form of list of dictionary \n",
    "#cFlow = []    ##Store the current flow (all details)\n",
    "count = 0\n",
    "fc = 0\n",
    "startTime = None   ##Start Time of each flow to implement timeout\n",
    "SF2 = SF2.sort(['UFid','Time'])\n",
    "print 'Done Sorting'\n",
    "for x in SF2:\n",
    "    if count%500000 == 0:\n",
    "        print 'Running '+str(count)+' Done !'\n",
    "        \n",
    "    count = count+1\n",
    "    \n",
    "    if prev is None:\n",
    "        if startTime is None:\n",
    "            startTime = x['Time']\n",
    "        Flow.append(FlowNo)\n",
    "        \n",
    "        prev = x['UFid']\n",
    "    elif compareUF(x['UFid'],prev):\n",
    "        if x['tcp_Flags']&1:\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = None\n",
    "            FlowNo = FlowNo + 1\n",
    "\n",
    "        elif x['Time']-startTime>=3600:\n",
    "            FlowNo = FlowNo + 1\n",
    "            Flow.append(FlowNo)\n",
    "            prev = None\n",
    "            startTime = x['Time']\n",
    "            \n",
    "        else:\n",
    "            \n",
    "            Flow.append(FlowNo)\n",
    "            prev = x['UFid']\n",
    "\n",
    "    else:\n",
    "        FlowNo = FlowNo + 1\n",
    "        Flow.append(FlowNo)\n",
    "        prev = x['UFid']\n",
    "        startTime = x['Time']\n",
    "\n",
    "\n",
    "print len(gl.SArray(Flow).unique())\n",
    "\n",
    "\n",
    "SF2['Flow'] = gl.SArray(Flow)\n",
    "temp = SF2.groupby('Flow',{\n",
    "            'Count':gl.aggregate.COUNT()\n",
    "        })\n",
    "len(temp[temp['Count']>1])\n",
    "\n",
    "\n",
    "temp = SF2.groupby('Flow',{\n",
    "        'NumBots' : gl.aggregate.SUM('isBot')\n",
    "    })\n",
    "NumBotFlows = len(temp[temp['NumBots']>1])\n",
    "print NumBotFlows, NumBotFlows*1.0/len(SF2['Flow'].unique()) \n",
    "\n",
    "SF2['FlowNo.'] = gl.SArray(Flow)\n",
    "\n",
    "import pickle\n",
    "pickle.dump(Flow,open('Flow.pkl','w'))\n",
    "\n",
    "SF2.save('ISCX_Botnet-Testing_Ports_Only_Sorted_Flow_BD.csv')\n",
    "\n",
    "\n",
    "def tfn(x):\n",
    "    if 'udp' in x.split(':'):\n",
    "        return 1\n",
    "    return 0\n",
    "SF2['hasUDP'] = SF2['Protocols in frame'].apply(lambda x:tfn(x))\n",
    "\n",
    "\n",
    "## Ratio of incoming to outgoing packets\n",
    "temp = SF2.groupby('FlowNo.',{\n",
    "        'NumForward' : gl.aggregate.SUM('Forward'),\n",
    "        'Total' : gl.aggregate.COUNT()\n",
    "    })\n",
    "temp['IOPR']= temp.apply(lambda x: ((x['Total']-x['NumForward'])*1.0)/x['NumForward'] if x['NumForward'] !=0 else (-1) )\n",
    "temp = temp['FlowNo.','IOPR']\n",
    "\n",
    "\n",
    "SF2 = SF2.join(temp,on='FlowNo.')\n",
    "SF2.head\n",
    "\n",
    "## First Packet Length\n",
    "FlowFeatures = ['Source','Destination','Source Port','Destination Port','Protocol']\n",
    "FPL = SF2.groupby(['FlowNo.'],{\n",
    "        'Time':gl.aggregate.MIN('Time')\n",
    "    })\n",
    "print len(FPL)\n",
    "FPL = FPL.join(SF2,on =['FlowNo.','Time'])[['FlowNo.','Length']].unique()\n",
    "FPL = FPL.groupby(['FlowNo.'],{\n",
    "        'FPL':gl.aggregate.AVG('Length')\n",
    "    })\n",
    "print len(FPL)\n",
    "\n",
    "\n",
    "# In[33]:\n",
    "\n",
    "SF2 = SF2.join(FPL, on ='FlowNo.')\n",
    "del(FPL)\n",
    "\n",
    "\n",
    "# ## 18/10/2016\n",
    "\n",
    "# In[35]:\n",
    "\n",
    "## Number of packets per flow\n",
    "temp = SF2.groupby(['FlowNo.'],{\n",
    "        'NumPackets':gl.aggregate.COUNT()\n",
    "    })\n",
    "print temp.head(3)\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "del(temp)\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "## Number of bytes exchanged\n",
    "temp = SF2.groupby(['FlowNo.'],{\n",
    "        'BytesEx':gl.aggregate.SUM('Length')\n",
    "    })\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "del(temp)\n",
    "\n",
    "\n",
    "# In[38]:\n",
    "\n",
    "## Standard deviation of packet length\n",
    "temp = SF2.groupby(['FlowNo.'],{\n",
    "        'StdDevLen':gl.aggregate.STDV('Length')\n",
    "    })\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "del(temp)\n",
    "\n",
    "\n",
    "# In[40]:\n",
    "\n",
    "## Same length packet ratio\n",
    "temp2 = SF2.groupby(['FlowNo.'],{\n",
    "        'SameLenPktRatio':gl.aggregate.COUNT_DISTINCT('Length')\n",
    "    })\n",
    "##temp from number of packets computation\n",
    "temp = SF2.groupby(['FlowNo.'],{\n",
    "        'NumPackets':gl.aggregate.COUNT()\n",
    "    })\n",
    "temp = temp.join(temp2,on='FlowNo.')\n",
    "temp['SameLenPktRatio'] = temp['SameLenPktRatio']*1.0/temp['NumPackets']\n",
    "temp2 = None\n",
    "temp = temp[['FlowNo.','SameLenPktRatio']]\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[41]:\n",
    "\n",
    "## Duration of flow\n",
    "timeF = SF2.groupby(['FlowNo.'],{\n",
    "        'startTime':gl.aggregate.MIN('Time'),\n",
    "        'endTime':gl.aggregate.MAX('Time')\n",
    "    })\n",
    "timeF['Duration'] = timeF['endTime'] - timeF['startTime']\n",
    "timeF = timeF[['FlowNo.','Duration']]\n",
    "SF2 = SF2.join(timeF, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[45]:\n",
    "\n",
    "sorted(SF2.column_names())\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# In[50]:\n",
    "\n",
    "features = ['Answer RRs',\n",
    " 'BytesEx',\n",
    " 'Destination',\n",
    " 'Destination Port',\n",
    " 'Differentiated Services Field',\n",
    " 'Duration',\n",
    " 'FPL',\n",
    " 'IP_Flags',\n",
    " 'Info',\n",
    " 'Length',\n",
    " 'Next sequence number',\n",
    " 'No.',\n",
    " 'NumPackets',\n",
    " 'Protocol',\n",
    " 'Protocols in frame',\n",
    " 'SameLenPktRatio',\n",
    " 'Sequence number',\n",
    " 'Source',\n",
    " 'Source Port',\n",
    " 'StdDevLen',\n",
    " 'TCP Segment Len',\n",
    " 'Time',\n",
    " 'Time to live',\n",
    " 'isBot',\n",
    " 'tcp_Flags',\n",
    " 'FlowNo.',\n",
    " 'udp_Length',\n",
    " 'IOPR']\n",
    "SF2 = SF2[features]\n",
    "\n",
    "\n",
    "# In[52]:\n",
    "\n",
    "## Average packets per second\n",
    "temp =  SF2.groupby(['FlowNo.'],{\n",
    "        'NumPackets':gl.aggregate.COUNT()\n",
    "    })\n",
    "temp = temp.join(timeF,on=['FlowNo.'])\n",
    "temp['AvgPktPerSec'] = temp.apply(lambda x:0.0 if x['Duration'] == 0.0 else x['NumPackets']*1.0/x['Duration'])\n",
    "temp = temp[['FlowNo.','AvgPktPerSec']]\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[53]:\n",
    "\n",
    "##Average Bits Per Second\n",
    "temp = SF2.groupby(['FlowNo.'],{\n",
    "        'BytesEx':gl.aggregate.SUM('Length')\n",
    "    })\n",
    "temp = temp.join(timeF,on=['FlowNo.'])\n",
    "temp['BitsPerSec'] = temp.apply(lambda x:0.0 if x['Duration'] == 0.0 else x['BytesEx']*8.0/x['Duration'])\n",
    "temp = temp[['FlowNo.','BitsPerSec']]\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[55]:\n",
    "\n",
    "## Average Packet Lentgth\n",
    "temp = SF2.groupby(['FlowNo.'],{\n",
    "        'APL':gl.aggregate.AVG('Length')\n",
    "    })\n",
    "SF2 = SF2.join(temp, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "## Number of Reconnects, sort FlowNo, SeqNo\n",
    "\n",
    "\n",
    "# In[56]:\n",
    "\n",
    "def tfn(x):\n",
    "    if 'udp' in x.split(':') or 'tcp' in x.split(':'):\n",
    "        return 1\n",
    "    return 0\n",
    "temp = list(SF2['Protocols in frame'].apply(lambda x:tfn(x)))\n",
    "\n",
    "\n",
    "# In[57]:\n",
    "\n",
    "len(temp)\n",
    "\n",
    "\n",
    "# In[58]:\n",
    "\n",
    "sum(temp)\n",
    "\n",
    "\n",
    "# In[60]:\n",
    "\n",
    "\n",
    "\n",
    "# In[62]:\n",
    "\n",
    "print len(SF2[SF2['Protocols in frame']=='eth:ethertype:ip:icmp:ip:tcp:http:urlencoded-form']),len(SF2[SF2['Protocols in frame']=='eth:ethertype:ip:icmp:ip:tcp']),len(SF2[SF2['Protocols in frame']=='eth:ethertype:ip:icmp:ip:tcp:http:data'])\n",
    "\n",
    "\n",
    "# In[64]:\n",
    "\n",
    "## Inter arrival time\n",
    "SF2['IAT'] = 0\n",
    "SF2 = SF2.sort(['FlowNo.','Time'])\n",
    "prev = None\n",
    "prevT = None\n",
    "li = []\n",
    "for x in SF2:\n",
    "    if prev is None or x['FlowNo.']!= prev:\n",
    "        li.append(0)\n",
    "    else:\n",
    "        li.append(x['Time']-prevT)        \n",
    "    prev = x['FlowNo.']\n",
    "    prevT = x['Time']\n",
    "SF2['IAT'] = gl.SArray(li)\n",
    "\n",
    "\n",
    "# In[65]:\n",
    "\n",
    "len(SF2)\n",
    "\n",
    "\n",
    "# In[66]:\n",
    "\n",
    "print len(SF2[SF2['Protocols in frame']=='eth:ipx'])\n",
    "\n",
    "\n",
    "# In[67]:\n",
    "\n",
    "SF2.save('Bidirectional_Test_Bot_features_till_IAT.csv')\n",
    "\n",
    "\n",
    "# # Is Null feature\n",
    "\n",
    "# ### Number of TCP Null packets\n",
    "\n",
    "# In[68]:\n",
    "\n",
    "\n",
    "# In[73]:\n",
    "\n",
    "## Null Packets handling\n",
    "def checkNull(x):\n",
    "    if(x['TCP Segment Len']=='0' or x['udp_Length']==8 ):\n",
    "        return 1\n",
    "    elif('ipx' in x['Protocols in frame'].split(':')):\n",
    "        l = x['Length'] - 30\n",
    "        if('eth' in x['Protocols in frame'].split(':')):\n",
    "            l = l - 14\n",
    "        if('ethtype' in x['Protocols in frame'].split(':')):\n",
    "            l = l - 2\n",
    "        if('llc' in x['Protocols in frame'].split(':')):\n",
    "            l = l - 8\n",
    "        if(l==0 or l==-1):\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "\n",
    "# In[74]:\n",
    "\n",
    "SF2['isNull'] = SF2.apply(lambda x:checkNull(x))\n",
    "\n",
    "\n",
    "# In[75]:\n",
    "\n",
    "\n",
    "\n",
    "# In[76]:\n",
    "\n",
    "NPEx = SF2.groupby(['FlowNo.'],{\n",
    "        'NPEx':gl.aggregate.SUM('isNull')\n",
    "    })\n",
    "SF2 = SF2.join(NPEx, on ='FlowNo.')\n",
    "\n",
    "\n",
    "# ### Number of Reconnects - considering only TCP reconnects, using sequence number\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "\n",
    "# In[79]:\n",
    "\n",
    "recon = SF2[SF2['Sequence number']!=''].groupby(['FlowNo.'],{\n",
    "        'total_seq_no.' : gl.aggregate.COUNT('Sequence number'),\n",
    "        'distinct_seq_no.' : gl.aggregate.COUNT_DISTINCT('Sequence number')\n",
    "    })\n",
    "recon['reconnects'] = recon['total_seq_no.'] - recon['distinct_seq_no.']\n",
    "recon.head()\n",
    "recon = recon[['FlowNo.','reconnects']]\n",
    "SF2 = SF2.join(recon,on='FlowNo.',how='left')\n",
    "len(SF2)\n",
    "\n",
    "\n",
    "# In[81]:\n",
    "\n",
    "#To identify records where reconnect check was not applied like UDP etc\n",
    "SF2.fillna('reconnects',-1)\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "#SF2[SF2['FlowNo.']==79732]['Info']\n",
    "\n",
    "\n",
    "# A lot of these reconnects can be simple retransmissions - due to out of order/timeout etcb\n",
    "\n",
    "# ### 22-10-2016\n",
    "\n",
    "# In[83]:\n",
    "\n",
    "\n",
    "\n",
    "# In[15]:\n",
    "\n",
    "print min(SF2['Time']),max(SF2['Time'])\n",
    "\n",
    "\n",
    "# # 28/10/16\n",
    "\n",
    "# Bidirectional Flowscombining all features and labels\n",
    "\n",
    "# In[84]:\n",
    "\n",
    "\n",
    "# In[85]:\n",
    "\n",
    "SF2.save('Bidirectional_Test_Botnet_all_features.csv')\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "#SF2_valid_Test, SF2_test = SF2.random_split(.8, seed=5)\n",
    "#SF2_valid, SF2_Test = SF2_valid_Test.random_split(.2, seed=5)\n",
    "#X_Test, y_Test = SF2_Test.drop('isBot'), SF2_Test['isBot']\n",
    "#X_valid, y_valid = SF2_valid_Test\n",
    "#X_Test_valid, y_Test_valid = X[:3475000], y[:3475000]\n",
    "#X_test, y_test = X[3475000:], y[3475000:]\n",
    "\n",
    "\n",
    "\n",
    "# In[89]:\n",
    "\n",
    "features = SF2.column_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Answer RRs',\n",
       " 'BytesEx',\n",
       " 'Destination',\n",
       " 'Destination Port',\n",
       " 'Differentiated Services Field',\n",
       " 'Duration',\n",
       " 'FPL',\n",
       " 'IP_Flags',\n",
       " 'Info',\n",
       " 'Length',\n",
       " 'Next sequence number',\n",
       " 'No.',\n",
       " 'NumPackets',\n",
       " 'Protocol',\n",
       " 'Protocols in frame',\n",
       " 'SameLenPktRatio',\n",
       " 'Sequence number',\n",
       " 'Source',\n",
       " 'Source Port',\n",
       " 'StdDevLen',\n",
       " 'TCP Segment Len',\n",
       " 'Time',\n",
       " 'Time to live',\n",
       " 'isBot',\n",
       " 'tcp_Flags',\n",
       " 'FlowNo.',\n",
       " 'udp_Length',\n",
       " 'IOPR',\n",
       " 'AvgPktPerSec',\n",
       " 'BitsPerSec',\n",
       " 'APL',\n",
       " 'IAT',\n",
       " 'isNull',\n",
       " 'NPEx',\n",
       " 'reconnects']"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features =SF2.column_names()\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'features' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-72-0731e29a81c2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfeatures\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'features' is not defined"
     ]
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BytesEx',\n",
       " 'Destination',\n",
       " 'Destination Port',\n",
       " 'Differentiated Services Field',\n",
       " 'Duration',\n",
       " 'FPL',\n",
       " 'Info',\n",
       " 'Length',\n",
       " 'NumPackets',\n",
       " 'Protocol',\n",
       " 'SameLenPktRatio',\n",
       " 'Source',\n",
       " 'Source Port',\n",
       " 'StdDevLen',\n",
       " 'TCP Segment Len',\n",
       " 'Time to live',\n",
       " 'FlowNo.',\n",
       " 'udp_Length',\n",
       " 'IOPR',\n",
       " 'AvgPktPerSec',\n",
       " 'BitsPerSec',\n",
       " 'APL',\n",
       " 'IAT',\n",
       " 'isNull',\n",
       " 'NPEx',\n",
       " 'reconnects']"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myfeatures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "15-11-2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "SF = gl.SFrame.read_csv('Bidirectional_Train_Botnet_all_features.csv',verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9360525"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SF['Forward'] = SF.apply(lambda x: 1 if x['Source']>x['Destination'] else 0 )\n",
    "temp = SF.groupby('FlowNo.',{\n",
    "        'NumForward' : gl.aggregate.SUM('Forward'),\n",
    "        \n",
    "    })\n",
    "\n",
    "SF = SF.join(temp,on='FlowNo.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method SFrame.head of Columns:\n",
       "\tAnswer RRs\tstr\n",
       "\tBytesEx\tint\n",
       "\tDestination\tstr\n",
       "\tDestination Port\tint\n",
       "\tDifferentiated Services Field\tstr\n",
       "\tDuration\tfloat\n",
       "\tFPL\tint\n",
       "\tIP_Flags\tstr\n",
       "\tInfo\tstr\n",
       "\tLength\tint\n",
       "\tNext sequence number\tstr\n",
       "\tNo.\tint\n",
       "\tNumPackets\tint\n",
       "\tProtocol\tstr\n",
       "\tProtocols in frame\tstr\n",
       "\tSameLenPktRatio\tfloat\n",
       "\tSequence number\tstr\n",
       "\tSource\tstr\n",
       "\tSource Port\tint\n",
       "\tStdDevLen\tfloat\n",
       "\tTCP Segment Len\tint\n",
       "\tTime\tint\n",
       "\tTime to live\tstr\n",
       "\tisBot\tint\n",
       "\ttcp_Flags\tint\n",
       "\tFlowNo.\tint\n",
       "\tudp_Length\tstr\n",
       "\tIOPR\tfloat\n",
       "\tAvgPktPerSec\tfloat\n",
       "\tBitsPerSec\tfloat\n",
       "\tAPL\tfloat\n",
       "\tIAT\tfloat\n",
       "\tisNull\tint\n",
       "\tNPEx\tint\n",
       "\treconnects\tint\n",
       "\tForward\tint\n",
       "\tNumForward\tint\n",
       "\n",
       "Rows: 9360525\n",
       "\n",
       "Data:\n",
       "+------------+---------+-----------------+------------------+-------------------------------+\n",
       "| Answer RRs | BytesEx |   Destination   | Destination Port | Differentiated Services Field |\n",
       "+------------+---------+-----------------+------------------+-------------------------------+\n",
       "|            |   372   |   1.22.16.143   |       3128       |              0x00             |\n",
       "|            |   372   |   1.22.16.143   |       3128       |              0x00             |\n",
       "|            |   372   |   1.22.16.143   |       3128       |              0x00             |\n",
       "|            |   372   |   1.22.16.143   |       3128       |              0x00             |\n",
       "|            |   372   |   1.22.16.143   |       3128       |              0x00             |\n",
       "|            |   372   |   1.22.16.143   |       3128       |              0x00             |\n",
       "|            |   372   | 109.200.239.148 |       3128       |              0x00             |\n",
       "|            |   372   | 109.200.239.148 |       3128       |              0x00             |\n",
       "|            |   372   | 109.200.239.148 |       3128       |              0x00             |\n",
       "|            |   372   | 109.200.239.148 |       3128       |              0x00             |\n",
       "+------------+---------+-----------------+------------------+-------------------------------+\n",
       "+----------+-----+----------+-------------------------------+--------+----------------------+\n",
       "| Duration | FPL | IP_Flags |              Info             | Length | Next sequence number |\n",
       "+----------+-----+----------+-------------------------------+--------+----------------------+\n",
       "| 9.01305  |  62 |   0x02   | 1085  >  3128 [SYN] Seq=0 ... |   62   |                      |\n",
       "| 9.01305  |  62 |   0x02   | [TCP Out-Of-Order] 1085  >... |   62   |                      |\n",
       "| 9.01305  |  62 |   0x02   | [TCP Retransmission] 1085 ... |   62   |                      |\n",
       "| 9.01305  |  62 |   0x02   | [TCP Retransmission] 1085 ... |   62   |                      |\n",
       "| 9.01305  |  62 |   0x02   | [TCP Retransmission] 1085 ... |   62   |                      |\n",
       "| 9.01305  |  62 |   0x02   | [TCP Retransmission] 1085 ... |   62   |                      |\n",
       "| 8.94841  |  62 |   0x02   | 1066  >  3128 [SYN] Seq=0 ... |   62   |                      |\n",
       "| 8.94841  |  62 |   0x02   | [TCP Out-Of-Order] 1066  >... |   62   |                      |\n",
       "| 8.94841  |  62 |   0x02   | [TCP Retransmission] 1066 ... |   62   |                      |\n",
       "| 8.94841  |  62 |   0x02   | [TCP Retransmission] 1066 ... |   62   |                      |\n",
       "+----------+-----+----------+-------------------------------+--------+----------------------+\n",
       "+--------+------------+----------+----------------------+-----------------+-----------------+\n",
       "|  No.   | NumPackets | Protocol |  Protocols in frame  | SameLenPktRatio | Sequence number |\n",
       "+--------+------------+----------+----------------------+-----------------+-----------------+\n",
       "| 819351 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819352 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819384 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819385 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819616 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819617 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819151 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819152 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819178 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "| 819179 |     6      |   TCP    | eth:ethertype:ip:tcp |     0.166667    |        0        |\n",
       "+--------+------------+----------+----------------------+-----------------+-----------------+\n",
       "+---------------+-------------+-----------+-----+\n",
       "|     Source    | Source Port | StdDevLen | ... |\n",
       "+---------------+-------------+-----------+-----+\n",
       "| 147.32.84.160 |     1085    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1085    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1085    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1085    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1085    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1085    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1066    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1066    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1066    |    0.0    | ... |\n",
       "| 147.32.84.160 |     1066    |    0.0    | ... |\n",
       "+---------------+-------------+-----------+-----+\n",
       "[9360525 rows x 37 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method SFrame.head of Columns:\n",
       "\tAnswer RRs\tstr\n",
       "\tBytesEx\tint\n",
       "\tDestination\tstr\n",
       "\tDestination Port\tstr\n",
       "\tDifferentiated Services Field\tstr\n",
       "\tDuration\tfloat\n",
       "\tFPL\tint\n",
       "\tIP_Flags\tstr\n",
       "\tInfo\tstr\n",
       "\tLength\tint\n",
       "\tNext sequence number\tstr\n",
       "\tNo.\tint\n",
       "\tNumPackets\tint\n",
       "\tProtocol\tstr\n",
       "\tProtocols in frame\tstr\n",
       "\tSameLenPktRatio\tfloat\n",
       "\tSequence number\tstr\n",
       "\tSource\tstr\n",
       "\tSource Port\tstr\n",
       "\tStdDevLen\tfloat\n",
       "\tTCP Segment Len\tstr\n",
       "\tTime\tfloat\n",
       "\tTime to live\tstr\n",
       "\tisBot\tint\n",
       "\ttcp_Flags\tint\n",
       "\tFlowNo.\tint\n",
       "\tudp_Length\tstr\n",
       "\tIOPR\tint\n",
       "\tAvgPktPerSec\tfloat\n",
       "\tBitsPerSec\tfloat\n",
       "\tAPL\tfloat\n",
       "\tIAT\tfloat\n",
       "\tisNull\tint\n",
       "\tNPEx\tint\n",
       "\treconnects\tstr\n",
       "\n",
       "Rows: 4676319\n",
       "\n",
       "Data:\n",
       "+------------+---------+-----------------------+------------------+-------------------------------+\n",
       "| Answer RRs | BytesEx |      Destination      | Destination Port | Differentiated Services Field |\n",
       "+------------+---------+-----------------------+------------------+-------------------------------+\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "|            |  27511  | 00000000.ffffffffffff |      0x0452      |                               |\n",
       "+------------+---------+-----------------------+------------------+-------------------------------+\n",
       "+----------+-----+----------+-------------------------------+--------+----------------------+\n",
       "| Duration | FPL | IP_Flags |              Info             | Length | Next sequence number |\n",
       "+----------+-----+----------+-------------------------------+--------+----------------------+\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  113   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  110   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  118   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  110   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  113   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  110   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  118   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  110   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  113   |                      |\n",
       "| 3596.95  | 113 |          | General Response[Packet si... |  110   |                      |\n",
       "+----------+-----+----------+-------------------------------+--------+----------------------+\n",
       "+---------+------------+----------+--------------------+-----------------+-----------------+\n",
       "|   No.   | NumPackets | Protocol | Protocols in frame | SameLenPktRatio | Sequence number |\n",
       "+---------+------------+----------+--------------------+-----------------+-----------------+\n",
       "| 1089192 |    244     | IPX SAP  |    eth:llc:ipx     |    0.0122951    |                 |\n",
       "| 1089193 |    244     | IPX SAP  | eth:ethertype:ipx  |    0.0122951    |                 |\n",
       "| 1089194 |    244     | IPX SAP  |    eth:llc:ipx     |    0.0122951    |                 |\n",
       "| 1089195 |    244     | IPX SAP  |      eth:ipx       |    0.0122951    |                 |\n",
       "| 1089509 |    244     | IPX SAP  |    eth:llc:ipx     |    0.0122951    |                 |\n",
       "| 1089510 |    244     | IPX SAP  | eth:ethertype:ipx  |    0.0122951    |                 |\n",
       "| 1089511 |    244     | IPX SAP  |    eth:llc:ipx     |    0.0122951    |                 |\n",
       "| 1089512 |    244     | IPX SAP  |      eth:ipx       |    0.0122951    |                 |\n",
       "| 1089966 |    244     | IPX SAP  |    eth:llc:ipx     |    0.0122951    |                 |\n",
       "| 1089967 |    244     | IPX SAP  | eth:ethertype:ipx  |    0.0122951    |                 |\n",
       "+---------+------------+----------+--------------------+-----------------+-----------------+\n",
       "+-----------------------+-------------+-----------+-----+\n",
       "|         Source        | Source Port | StdDevLen | ... |\n",
       "+-----------------------+-------------+-----------+-----+\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "| 00000000.00040065437c |    0x83c2   |  3.26917  | ... |\n",
       "+-----------------------+-------------+-----------+-----+\n",
       "[4676319 rows x 35 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Test = gl.SFrame.read_csv('Bidirectional_Test_Botnet_all_features.csv',verbose=False)\n",
    "Test.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for col in ['Source Port','Destination Port']:\n",
    "    SF[col] = SF[col].apply(lambda x: str(x))\n",
    "    Test[col] = Test[col].apply(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Test['Forward'] = Test.apply(lambda x: 1 if x['Source']>x['Destination'] else 0 )\n",
    "temp = Test.groupby('FlowNo.',{\n",
    "        'NumForward' : gl.aggregate.SUM('Forward'),\n",
    "        \n",
    "    })\n",
    "\n",
    "Test= Test.join(temp,on='FlowNo.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Test = Test.groupby('FlowNo.',{\n",
    "        'Answer RRs': gl.aggregate.SELECT_ONE('Answer RRs'),\n",
    "        'BytesEx' : gl.aggregate.SELECT_ONE('BytesEx'),\n",
    "        'Destination' : gl.aggregate.SELECT_ONE('Destination'),\n",
    "        'Destination Port' : gl.aggregate.SELECT_ONE('Destination Port'),\n",
    "        'Differentiated Services Field' : gl.aggregate.SELECT_ONE('Differentiated Services Field'),\n",
    "        'Duration' : gl.aggregate.SELECT_ONE('Duration'),\n",
    "        'FPL' : gl.aggregate.SELECT_ONE('FPL'),\n",
    "        'IP_Flags' : gl.aggregate.SELECT_ONE('IP_Flags'),\n",
    "        'Info' : gl.aggregate.SELECT_ONE('Info'),\n",
    "        'Length' : gl.aggregate.SELECT_ONE('Length'),\n",
    "        'Next sequence number' : gl.aggregate.SELECT_ONE('Next sequence number'),\n",
    "        'No.' : gl.aggregate.SELECT_ONE('No.'),\n",
    "        'NumPackets' : gl.aggregate.SELECT_ONE('NumPackets'),\n",
    "        'Protocol' : gl.aggregate.SELECT_ONE('Protocol'),\n",
    "        'Protocols in frame' : gl.aggregate.SELECT_ONE('Protocols in frame'),\n",
    "        'SameLenPktRatio' : gl.aggregate.SELECT_ONE('SameLenPktRatio'),\n",
    "        'Sequence number' : gl.aggregate.SELECT_ONE('Sequence number'),\n",
    "        'Source' : gl.aggregate.SELECT_ONE('Source'),\n",
    "        'Source Port' : gl.aggregate.SELECT_ONE('Source Port'),\n",
    "        'StdDevLen' : gl.aggregate.SELECT_ONE('StdDevLen'),\n",
    "        'IAT' : gl.aggregate.SELECT_ONE('IAT'),\n",
    "        'isNull' : gl.aggregate.SELECT_ONE('isNull'),\n",
    "        'NPEx' : gl.aggregate.SELECT_ONE('NPEx'),\n",
    "        'reconnects' : gl.aggregate.SELECT_ONE('reconnects'),\n",
    "        'APL' : gl.aggregate.SELECT_ONE('APL'),\n",
    "        'BitsPerSec' : gl.aggregate.SELECT_ONE('BitsPerSec'),\n",
    "        'AvgPktPerSec' : gl.aggregate.SELECT_ONE('AvgPktPerSec'),\n",
    "        'udp_Length' : gl.aggregate.SELECT_ONE('udp_Length'),\n",
    "        'tcp_Flags' : gl.aggregate.SELECT_ONE('tcp_Flags'),\n",
    "        'isBot' : gl.aggregate.SELECT_ONE('isBot'),\n",
    "        'Time to live' : gl.aggregate.SELECT_ONE('Time to live'),\n",
    "        'Time' : gl.aggregate.SELECT_ONE('Time'),\n",
    "        'TCP Segment Len' : gl.aggregate.SELECT_ONE('TCP Segment Len'),\n",
    "        'IOPR' : gl.aggregate.SELECT_ONE('IOPR'),\n",
    "        'NumForward' : gl.aggregate.SELECT_ONE('NumForward')\n",
    "    })\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "SF = SF.groupby('FlowNo.',{\n",
    "        'Answer RRs': gl.aggregate.SELECT_ONE('Answer RRs'),\n",
    "        'BytesEx' : gl.aggregate.SELECT_ONE('BytesEx'),\n",
    "        'Destination' : gl.aggregate.SELECT_ONE('Destination'),\n",
    "        'Destination Port' : gl.aggregate.SELECT_ONE('Destination Port'),\n",
    "        'Differentiated Services Field' : gl.aggregate.SELECT_ONE('Differentiated Services Field'),\n",
    "        'Duration' : gl.aggregate.SELECT_ONE('Duration'),\n",
    "        'FPL' : gl.aggregate.SELECT_ONE('FPL'),\n",
    "        'IP_Flags' : gl.aggregate.SELECT_ONE('IP_Flags'),\n",
    "        'Info' : gl.aggregate.SELECT_ONE('Info'),\n",
    "        'Length' : gl.aggregate.SELECT_ONE('Length'),\n",
    "        'Next sequence number' : gl.aggregate.SELECT_ONE('Next sequence number'),\n",
    "        'No.' : gl.aggregate.SELECT_ONE('No.'),\n",
    "        'NumPackets' : gl.aggregate.SELECT_ONE('NumPackets'),\n",
    "        'Protocol' : gl.aggregate.SELECT_ONE('Protocol'),\n",
    "        'Protocols in frame' : gl.aggregate.SELECT_ONE('Protocols in frame'),\n",
    "        'SameLenPktRatio' : gl.aggregate.SELECT_ONE('SameLenPktRatio'),\n",
    "        'Sequence number' : gl.aggregate.SELECT_ONE('Sequence number'),\n",
    "        'Source' : gl.aggregate.SELECT_ONE('Source'),\n",
    "        'Source Port' : gl.aggregate.SELECT_ONE('Source Port'),\n",
    "        'StdDevLen' : gl.aggregate.SELECT_ONE('StdDevLen'),\n",
    "        'IAT' : gl.aggregate.SELECT_ONE('IAT'),\n",
    "        'isNull' : gl.aggregate.SELECT_ONE('isNull'),\n",
    "        'NPEx' : gl.aggregate.SELECT_ONE('NPEx'),\n",
    "        'reconnects' : gl.aggregate.SELECT_ONE('reconnects'),\n",
    "        'APL' : gl.aggregate.SELECT_ONE('APL'),\n",
    "        'BitsPerSec' : gl.aggregate.SELECT_ONE('BitsPerSec'),\n",
    "        'AvgPktPerSec' : gl.aggregate.SELECT_ONE('AvgPktPerSec'),\n",
    "        'udp_Length' : gl.aggregate.SELECT_ONE('udp_Length'),\n",
    "        'tcp_Flags' : gl.aggregate.SELECT_ONE('tcp_Flags'),\n",
    "        'isBot' : gl.aggregate.SELECT_ONE('isBot'),\n",
    "        'Time to live' : gl.aggregate.SELECT_ONE('Time to live'),\n",
    "        'Time' : gl.aggregate.SELECT_ONE('Time'),\n",
    "        'TCP Segment Len' : gl.aggregate.SELECT_ONE('TCP Segment Len'),\n",
    "        'IOPR' : gl.aggregate.SELECT_ONE('IOPR'),\n",
    "        'NumForward' : gl.aggregate.SELECT_ONE('NumForward')\n",
    "    })\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "iplist = [\n",
    "    \"147.32.84.180\",\n",
    "    \"147.32.84.170\",\n",
    "    \"147.32.84.150\",\n",
    "    \"147.32.84.140\",\n",
    "    \"147.32.84.130\",\n",
    "    \"147.32.84.160\",\n",
    "    \"10.0.2.15\",\n",
    "    \"192.168.106.141\",\n",
    "    \"192.168.106.131\",\n",
    "    \"172.16.253.130\",\n",
    "    \"172.16.253.131\",\n",
    "    \"172.16.253.129\",\n",
    "    \"172.16.253.240\",\n",
    "    \"74.78.117.238\",\n",
    "    \"158.65.110.24\",\n",
    "    \"192.168.3.35\",\n",
    "    \"192.168.3.25\",\n",
    "    \"192.168.3.65\",\n",
    "    \"172.29.0.116\",\n",
    "    \"172.29.0.109\",\n",
    "    \"172.16.253.132\",\n",
    "    \"192.168.248.165\",\n",
    "    \"10.37.130.4\",\n",
    "    \"131.202.243.84\", \n",
    "    \"198.164.30.2\", \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \"192.168.5.122\", \"192.168.5.122\", \n",
    "    \"192.168.2.110\", \n",
    "    \"192.168.4.120\", \"192.168.1.103\",\n",
    "    \"192.168.2.113\", \"192.168.4.118\" , \"192.168.2.109\", \"192.168.2.105\", \"192.168.5.122\"]\n",
    "\n",
    "MasterBot = [ (\"192.168.2.112\", \"131.202.243.84\"), (\"192.168.5.122\", \"198.164.30.2\"), (\"192.168.2.110\", \"192.168.5.122\"),( \"192.168.4.118\", \"192.168.5.122\"), (\"192.168.2.113\", \"192.168.5.122\"), (\"192.168.1.103\", \"192.168.5.122\"), (\"192.168.4.120\", \"192.168.5.122\"), (\"192.168.2.112\", \"192.168.2.110\"), (\"192.168.2.112\", \"192.168.4.120\"), (\"192.168.2.112\", \"192.168.1.103\"), (\"192.168.2.112\", \"192.168.2.113\"), (\"192.168.2.112\", \"192.168.4.118\"), (\"192.168.2.112\", \"192.168.2.109\"), (\"192.168.2.112\", \"192.168.2.105\"), (\"192.168.1.105\", \"192.168.5.122\")]\n",
    "\n",
    "iplist = gl.SArray(iplist)\n",
    "iplist = iplist.unique() \n",
    "iplist = list(iplist)\n",
    "\n",
    "def fcheckIP(x):\n",
    "    if ((x['NumForward']==0) and (x['Destination'] in iplist) ) or ((x['NumForward']==x['NumPackets']) and (x['Source'] in iplist) ):\n",
    "        return 1\n",
    "    elif ((x['Source'] in iplist) or (x['Destination'] in iplist)) and (x['NumForward']!=0) and (x['NumForward']!=x['NumPackets']):\n",
    "        return 1\n",
    "    else:\n",
    "        if ((x['Source'],x['Destination'])  in MasterBot ) or ((x['Destination'],x['Source'])  in MasterBot ) :\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "        \n",
    "\n",
    "SF['isBot'] = SF[['Source','Destination','NumForward','NumPackets']].apply(lambda x: fcheckIP(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "185311 0.367648459261\n"
     ]
    }
   ],
   "source": [
    "temp = SF.groupby('FlowNo.',{\n",
    "        'NumBots' : gl.aggregate.SUM('isBot')\n",
    "    })\n",
    "NumBotFlows = len(temp[temp['NumBots']>=1])\n",
    "print NumBotFlows, NumBotFlows*1.0/len(SF['FlowNo.'].unique()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169686 0.477405087864\n"
     ]
    }
   ],
   "source": [
    "Test['isBot'] = Test[['Source','Destination','NumForward','NumPackets']].apply(lambda x: fcheckIP(x))\n",
    "\n",
    "temp = Test.groupby('FlowNo.',{\n",
    "        'NumBots' : gl.aggregate.SUM('isBot')\n",
    "    })\n",
    "NumBotFlows = len(temp[temp['NumBots']>=1])\n",
    "print NumBotFlows, NumBotFlows*1.0/len(Test['FlowNo.'].unique()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(169686, 355434)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Test[Test['isBot']==1]),len(Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(185311, 504044)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF[SF['isBot']==1]),len(SF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Test.save('Bidirectional_Botnet_Test_Final_Flow_Based_Features.csv')\n",
    "SF.save('Bidirectional_Botnet_Training_Final_Flow_Based_Features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "models = []\n",
    "\n",
    "myfeatures = Test.column_names()\n",
    "for x in ['isBot','Answer RRs','Sequence number','No.','IP_Flags','Next sequence number','Protocols in frame','Time','tcp_Flags','FlowNo.','udp_Length']:\n",
    "    #print x\n",
    "    #if x not in myfeatures:\n",
    "        #print x \n",
    "    myfeatures.remove(x)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Info',\n",
       " 'Destination',\n",
       " 'APL',\n",
       " 'AvgPktPerSec',\n",
       " 'Destination Port',\n",
       " 'TCP Segment Len',\n",
       " 'Differentiated Services Field',\n",
       " 'IOPR',\n",
       " 'Source',\n",
       " 'Length',\n",
       " 'IAT',\n",
       " 'reconnects',\n",
       " 'NumForward',\n",
       " 'Protocol',\n",
       " 'BytesEx',\n",
       " 'isNull',\n",
       " 'BitsPerSec',\n",
       " 'NumPackets',\n",
       " 'Source Port',\n",
       " 'StdDevLen',\n",
       " 'SameLenPktRatio',\n",
       " 'FPL',\n",
       " 'Duration',\n",
       " 'Time to live',\n",
       " 'NPEx']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myfeatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of flows having only destination to source, and source as malicious 247\n"
     ]
    }
   ],
   "source": [
    "l=0\n",
    "for x in iplist:\n",
    "    l=l+ len(Test[(Test['IOPR']==-1) & (Test['isBot']==1) & (Test['Source']==x)])\n",
    "print \"Number of flows having only destination to source, and source as malicious \" + str(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of flows having only destination to source, and source as malicious 2159\n"
     ]
    }
   ],
   "source": [
    "l=0\n",
    "for x in iplist:\n",
    "    l=l+ len(SF[(SF['IOPR']==-1) & (SF['isBot']==1) & (SF['Source']==x)])\n",
    "print \"Number of flows having only destination to source, and source as malicious \" + str(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of flows having only destination to source, and source as malicious 73443\n"
     ]
    }
   ],
   "source": [
    "l=0\n",
    "for x in iplist:\n",
    "    l=l+ len(SF[(SF['NumForward']==0) & (SF['isBot']==1) & (SF['Source']==x)])\n",
    "print \"Number of flows having only destination to source, and source as malicious \" + str(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50535"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF[(SF['IOPR']==0) & (SF['isBot']==1) ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "248083"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF[(SF['IOPR']==0) | (SF['IOPR']==-1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(92524, 355434)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Test[Test['IOPR']==-1]),len(Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(155107, 504044)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF[SF['IOPR']==-1]),len(SF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for x in ['Source','Source Port','Destination','Destination Port','Differentiated Services Field','TCP Segment Len','Length']:\n",
    "    myfeatures.remove(x)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "myfeatures1 = ['APL','IOPR','BitsPerSec','Duration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "myfeatures.remove('isNull')\n",
    "myfeatures.remove('Info')\n",
    "myfeatures.remove('Time to live')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94391"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Test[Test['reconnects']==''])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "myfeatures.remove('reconnects')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "myfeatures.remove('IOPR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['APL',\n",
       " 'AvgPktPerSec',\n",
       " 'IAT',\n",
       " 'NumForward',\n",
       " 'Protocol',\n",
       " 'BytesEx',\n",
       " 'BitsPerSec',\n",
       " 'NumPackets',\n",
       " 'StdDevLen',\n",
       " 'SameLenPktRatio',\n",
       " 'FPL',\n",
       " 'Duration',\n",
       " 'NPEx']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myfeatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(185748, 169686)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Test[Test['isBot']==0]),len(Test[Test['isBot']==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(318733, 185311)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF[SF['isBot']==0]),len(SF[SF['isBot']==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SF = SF[SF['NumPackets']>1]\n",
    "Test = Test[Test['NumPackets']>1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Boosted trees classifier:</pre>"
      ],
      "text/plain": [
       "Boosted trees classifier:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>--------------------------------------------------------</pre>"
      ],
      "text/plain": [
       "--------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of examples          : 323547</pre>"
      ],
      "text/plain": [
       "Number of examples          : 323547"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of classes           : 2</pre>"
      ],
      "text/plain": [
       "Number of classes           : 2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of feature columns   : 13</pre>"
      ],
      "text/plain": [
       "Number of feature columns   : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of unpacked features : 13</pre>"
      ],
      "text/plain": [
       "Number of unpacked features : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |</pre>"
      ],
      "text/plain": [
       "| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1         | 0.247143     | 0.744229          | 0.741615            | 0.587253          | 0.587057            |</pre>"
      ],
      "text/plain": [
       "| 1         | 0.247143     | 0.744229          | 0.741615            | 0.587253          | 0.587057            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2         | 0.450054     | 0.758112          | 0.757514            | 0.528094          | 0.527747            |</pre>"
      ],
      "text/plain": [
       "| 2         | 0.450054     | 0.758112          | 0.757514            | 0.528094          | 0.527747            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 3         | 0.679275     | 0.785462          | 0.783488            | 0.475099          | 0.475238            |</pre>"
      ],
      "text/plain": [
       "| 3         | 0.679275     | 0.785462          | 0.783488            | 0.475099          | 0.475238            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 4         | 0.903571     | 0.809660          | 0.811453            | 0.437680          | 0.437939            |</pre>"
      ],
      "text/plain": [
       "| 4         | 0.903571     | 0.809660          | 0.811453            | 0.437680          | 0.437939            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 5         | 1.122586     | 0.827889          | 0.828786            | 0.406961          | 0.407100            |</pre>"
      ],
      "text/plain": [
       "| 5         | 1.122586     | 0.827889          | 0.828786            | 0.406961          | 0.407100            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 6         | 1.371635     | 0.827840          | 0.828699            | 0.394319          | 0.394493            |</pre>"
      ],
      "text/plain": [
       "| 6         | 1.371635     | 0.827840          | 0.828699            | 0.394319          | 0.394493            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 10        | 2.218914     | 0.843729          | 0.843374            | 0.353636          | 0.354406            |</pre>"
      ],
      "text/plain": [
       "| 10        | 2.218914     | 0.843729          | 0.843374            | 0.353636          | 0.354406            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 11        | 2.426203     | 0.843405          | 0.843634            | 0.347964          | 0.348810            |</pre>"
      ],
      "text/plain": [
       "| 11        | 2.426203     | 0.843405          | 0.843634            | 0.347964          | 0.348810            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 15        | 3.329470     | 0.848229          | 0.847924            | 0.327205          | 0.328485            |</pre>"
      ],
      "text/plain": [
       "| 15        | 3.329470     | 0.848229          | 0.847924            | 0.327205          | 0.328485            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 20        | 4.361314     | 0.853449          | 0.853227            | 0.316099          | 0.317495            |</pre>"
      ],
      "text/plain": [
       "| 20        | 4.361314     | 0.853449          | 0.853227            | 0.316099          | 0.317495            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 25        | 5.459805     | 0.860119          | 0.859174            | 0.302305          | 0.304185            |</pre>"
      ],
      "text/plain": [
       "| 25        | 5.459805     | 0.860119          | 0.859174            | 0.302305          | 0.304185            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 30        | 6.520337     | 0.863893          | 0.862376            | 0.291576          | 0.293862            |</pre>"
      ],
      "text/plain": [
       "| 30        | 6.520337     | 0.863893          | 0.862376            | 0.291576          | 0.293862            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 35        | 7.559788     | 0.863878          | 0.862487            | 0.286247          | 0.289004            |</pre>"
      ],
      "text/plain": [
       "| 35        | 7.559788     | 0.863878          | 0.862487            | 0.286247          | 0.289004            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 40        | 8.620419     | 0.867787          | 0.865627            | 0.278352          | 0.281373            |</pre>"
      ],
      "text/plain": [
       "| 40        | 8.620419     | 0.867787          | 0.865627            | 0.278352          | 0.281373            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 45        | 9.733073     | 0.869524          | 0.867123            | 0.275034          | 0.278364            |</pre>"
      ],
      "text/plain": [
       "| 45        | 9.733073     | 0.869524          | 0.867123            | 0.275034          | 0.278364            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 50        | 10.845727    | 0.871425          | 0.867643            | 0.270850          | 0.274871            |</pre>"
      ],
      "text/plain": [
       "| 50        | 10.845727    | 0.871425          | 0.867643            | 0.270850          | 0.274871            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 51        | 11.064529    | 0.871737          | 0.867741            | 0.269267          | 0.273403            |</pre>"
      ],
      "text/plain": [
       "| 51        | 11.064529    | 0.871737          | 0.867741            | 0.269267          | 0.273403            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 55        | 11.942787    | 0.873184          | 0.868904            | 0.266661          | 0.271071            |</pre>"
      ],
      "text/plain": [
       "| 55        | 11.942787    | 0.873184          | 0.868904            | 0.266661          | 0.271071            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 60        | 13.082201    | 0.876185          | 0.872081            | 0.261644          | 0.266691            |</pre>"
      ],
      "text/plain": [
       "| 60        | 13.082201    | 0.876185          | 0.872081            | 0.261644          | 0.266691            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 65        | 14.134825    | 0.879038          | 0.873935            | 0.258372          | 0.263952            |</pre>"
      ],
      "text/plain": [
       "| 65        | 14.134825    | 0.879038          | 0.873935            | 0.258372          | 0.263952            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 70        | 15.115340    | 0.879427          | 0.874380            | 0.255779          | 0.261636            |</pre>"
      ],
      "text/plain": [
       "| 70        | 15.115340    | 0.879427          | 0.874380            | 0.255779          | 0.261636            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 75        | 16.167598    | 0.879795          | 0.874739            | 0.254682          | 0.260586            |</pre>"
      ],
      "text/plain": [
       "| 75        | 16.167598    | 0.879795          | 0.874739            | 0.254682          | 0.260586            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 80        | 17.202581    | 0.881717          | 0.876754            | 0.251409          | 0.257840            |</pre>"
      ],
      "text/plain": [
       "| 80        | 17.202581    | 0.881717          | 0.876754            | 0.251409          | 0.257840            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 85        | 18.258096    | 0.883334          | 0.877842            | 0.248231          | 0.255250            |</pre>"
      ],
      "text/plain": [
       "| 85        | 18.258096    | 0.883334          | 0.877842            | 0.248231          | 0.255250            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 90        | 19.321250    | 0.884672          | 0.878510            | 0.246448          | 0.254119            |</pre>"
      ],
      "text/plain": [
       "| 90        | 19.321250    | 0.884672          | 0.878510            | 0.246448          | 0.254119            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 95        | 20.427455    | 0.885439          | 0.878918            | 0.244752          | 0.252705            |</pre>"
      ],
      "text/plain": [
       "| 95        | 20.427455    | 0.885439          | 0.878918            | 0.244752          | 0.252705            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 100       | 21.470639    | 0.885936          | 0.879165            | 0.242859          | 0.251379            |</pre>"
      ],
      "text/plain": [
       "| 100       | 21.470639    | 0.885936          | 0.879165            | 0.242859          | 0.251379            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre>Boosted trees classifier:</pre>"
      ],
      "text/plain": [
       "Boosted trees classifier:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>--------------------------------------------------------</pre>"
      ],
      "text/plain": [
       "--------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of examples          : 323547</pre>"
      ],
      "text/plain": [
       "Number of examples          : 323547"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of classes           : 2</pre>"
      ],
      "text/plain": [
       "Number of classes           : 2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of feature columns   : 13</pre>"
      ],
      "text/plain": [
       "Number of feature columns   : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of unpacked features : 13</pre>"
      ],
      "text/plain": [
       "Number of unpacked features : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |</pre>"
      ],
      "text/plain": [
       "| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1         | 0.249280     | 0.743481          | 0.744458            | 0.587165          | 0.587371            |</pre>"
      ],
      "text/plain": [
       "| 1         | 0.249280     | 0.743481          | 0.744458            | 0.587165          | 0.587371            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2         | 0.466578     | 0.761067          | 0.762310            | 0.520105          | 0.520249            |</pre>"
      ],
      "text/plain": [
       "| 2         | 0.466578     | 0.761067          | 0.762310            | 0.520105          | 0.520249            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 3         | 0.685177     | 0.777154          | 0.776515            | 0.477320          | 0.477576            |</pre>"
      ],
      "text/plain": [
       "| 3         | 0.685177     | 0.777154          | 0.776515            | 0.477320          | 0.477576            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 4         | 0.945328     | 0.784288          | 0.781510            | 0.449469          | 0.449877            |</pre>"
      ],
      "text/plain": [
       "| 4         | 0.945328     | 0.784288          | 0.781510            | 0.449469          | 0.449877            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 5         | 1.167777     | 0.805246          | 0.804678            | 0.418323          | 0.418981            |</pre>"
      ],
      "text/plain": [
       "| 5         | 1.167777     | 0.805246          | 0.804678            | 0.418323          | 0.418981            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 6         | 1.405216     | 0.805642          | 0.804876            | 0.402008          | 0.402738            |</pre>"
      ],
      "text/plain": [
       "| 6         | 1.405216     | 0.805642          | 0.804876            | 0.402008          | 0.402738            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 10        | 2.348814     | 0.843046          | 0.841742            | 0.348716          | 0.350417            |</pre>"
      ],
      "text/plain": [
       "| 10        | 2.348814     | 0.843046          | 0.841742            | 0.348716          | 0.350417            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 11        | 2.569778     | 0.842777          | 0.841050            | 0.345308          | 0.347074            |</pre>"
      ],
      "text/plain": [
       "| 11        | 2.569778     | 0.842777          | 0.841050            | 0.345308          | 0.347074            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 15        | 3.432982     | 0.849673          | 0.847132            | 0.329087          | 0.331670            |</pre>"
      ],
      "text/plain": [
       "| 15        | 3.432982     | 0.849673          | 0.847132            | 0.329087          | 0.331670            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 20        | 4.537403     | 0.852600          | 0.850025            | 0.320025          | 0.322678            |</pre>"
      ],
      "text/plain": [
       "| 20        | 4.537403     | 0.852600          | 0.850025            | 0.320025          | 0.322678            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 25        | 5.690269     | 0.857276          | 0.855082            | 0.305907          | 0.308655            |</pre>"
      ],
      "text/plain": [
       "| 25        | 5.690269     | 0.857276          | 0.855082            | 0.305907          | 0.308655            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 30        | 6.759038     | 0.858935          | 0.856726            | 0.297186          | 0.300335            |</pre>"
      ],
      "text/plain": [
       "| 30        | 6.759038     | 0.858935          | 0.856726            | 0.297186          | 0.300335            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 35        | 7.933541     | 0.862283          | 0.860497            | 0.285811          | 0.289211            |</pre>"
      ],
      "text/plain": [
       "| 35        | 7.933541     | 0.862283          | 0.860497            | 0.285811          | 0.289211            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 40        | 8.986429     | 0.865862          | 0.863081            | 0.279406          | 0.283379            |</pre>"
      ],
      "text/plain": [
       "| 40        | 8.986429     | 0.865862          | 0.863081            | 0.279406          | 0.283379            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 45        | 10.091036    | 0.867633          | 0.864861            | 0.275366          | 0.279871            |</pre>"
      ],
      "text/plain": [
       "| 45        | 10.091036    | 0.867633          | 0.864861            | 0.275366          | 0.279871            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 50        | 11.206780    | 0.870108          | 0.867160            | 0.271203          | 0.275945            |</pre>"
      ],
      "text/plain": [
       "| 50        | 11.206780    | 0.870108          | 0.867160            | 0.271203          | 0.275945            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 51        | 11.434079    | 0.870149          | 0.867593            | 0.269368          | 0.274164            |</pre>"
      ],
      "text/plain": [
       "| 51        | 11.434079    | 0.870149          | 0.867593            | 0.269368          | 0.274164            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 55        | 12.362151    | 0.872547          | 0.869237            | 0.265779          | 0.270948            |</pre>"
      ],
      "text/plain": [
       "| 55        | 12.362151    | 0.872547          | 0.869237            | 0.265779          | 0.270948            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 60        | 13.476856    | 0.874290          | 0.870140            | 0.263100          | 0.268708            |</pre>"
      ],
      "text/plain": [
       "| 60        | 13.476856    | 0.874290          | 0.870140            | 0.263100          | 0.268708            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 65        | 14.581921    | 0.876173          | 0.871524            | 0.261050          | 0.267159            |</pre>"
      ],
      "text/plain": [
       "| 65        | 14.581921    | 0.876173          | 0.871524            | 0.261050          | 0.267159            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 70        | 15.690116    | 0.878552          | 0.873503            | 0.257120          | 0.263812            |</pre>"
      ],
      "text/plain": [
       "| 70        | 15.690116    | 0.878552          | 0.873503            | 0.257120          | 0.263812            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 75        | 16.770111    | 0.879316          | 0.873713            | 0.255015          | 0.261951            |</pre>"
      ],
      "text/plain": [
       "| 75        | 16.770111    | 0.879316          | 0.873713            | 0.255015          | 0.261951            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 80        | 17.831426    | 0.881829          | 0.875382            | 0.250920          | 0.258618            |</pre>"
      ],
      "text/plain": [
       "| 80        | 17.831426    | 0.881829          | 0.875382            | 0.250920          | 0.258618            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 85        | 18.901918    | 0.883838          | 0.877113            | 0.247887          | 0.256107            |</pre>"
      ],
      "text/plain": [
       "| 85        | 18.901918    | 0.883838          | 0.877113            | 0.247887          | 0.256107            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 90        | 19.998673    | 0.884487          | 0.877743            | 0.246723          | 0.255203            |</pre>"
      ],
      "text/plain": [
       "| 90        | 19.998673    | 0.884487          | 0.877743            | 0.246723          | 0.255203            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 95        | 21.154343    | 0.884944          | 0.878287            | 0.244870          | 0.253574            |</pre>"
      ],
      "text/plain": [
       "| 95        | 21.154343    | 0.884944          | 0.878287            | 0.244870          | 0.253574            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 100       | 22.219860    | 0.885188          | 0.878349            | 0.244164          | 0.253068            |</pre>"
      ],
      "text/plain": [
       "| 100       | 22.219860    | 0.885188          | 0.878349            | 0.244164          | 0.253068            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre>Boosted trees classifier:</pre>"
      ],
      "text/plain": [
       "Boosted trees classifier:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>--------------------------------------------------------</pre>"
      ],
      "text/plain": [
       "--------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of examples          : 323547</pre>"
      ],
      "text/plain": [
       "Number of examples          : 323547"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of classes           : 2</pre>"
      ],
      "text/plain": [
       "Number of classes           : 2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of feature columns   : 13</pre>"
      ],
      "text/plain": [
       "Number of feature columns   : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of unpacked features : 13</pre>"
      ],
      "text/plain": [
       "Number of unpacked features : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |</pre>"
      ],
      "text/plain": [
       "| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1         | 0.263471     | 0.743456          | 0.744903            | 0.587202          | 0.587177            |</pre>"
      ],
      "text/plain": [
       "| 1         | 0.263471     | 0.743456          | 0.744903            | 0.587202          | 0.587177            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2         | 0.502695     | 0.757850          | 0.758070            | 0.528075          | 0.528044            |</pre>"
      ],
      "text/plain": [
       "| 2         | 0.502695     | 0.757850          | 0.758070            | 0.528075          | 0.528044            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 3         | 0.733740     | 0.785008          | 0.784823            | 0.475102          | 0.475179            |</pre>"
      ],
      "text/plain": [
       "| 3         | 0.733740     | 0.785008          | 0.784823            | 0.475102          | 0.475179            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 4         | 0.966231     | 0.821204          | 0.820008            | 0.434707          | 0.434760            |</pre>"
      ],
      "text/plain": [
       "| 4         | 0.966231     | 0.821204          | 0.820008            | 0.434707          | 0.434760            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 5         | 1.211460     | 0.821504          | 0.820292            | 0.417252          | 0.417191            |</pre>"
      ],
      "text/plain": [
       "| 5         | 1.211460     | 0.821504          | 0.820292            | 0.417252          | 0.417191            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 6         | 1.433712     | 0.831910          | 0.831098            | 0.395554          | 0.395579            |</pre>"
      ],
      "text/plain": [
       "| 6         | 1.433712     | 0.831910          | 0.831098            | 0.395554          | 0.395579            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 10        | 2.408000     | 0.840521          | 0.839257            | 0.356644          | 0.357131            |</pre>"
      ],
      "text/plain": [
       "| 10        | 2.408000     | 0.840521          | 0.839257            | 0.356644          | 0.357131            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 11        | 2.644556     | 0.841757          | 0.840432            | 0.344068          | 0.344519            |</pre>"
      ],
      "text/plain": [
       "| 11        | 2.644556     | 0.841757          | 0.840432            | 0.344068          | 0.344519            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 15        | 3.515072     | 0.845624          | 0.844227            | 0.330888          | 0.331483            |</pre>"
      ],
      "text/plain": [
       "| 15        | 3.515072     | 0.845624          | 0.844227            | 0.330888          | 0.331483            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 20        | 4.621799     | 0.854652          | 0.852696            | 0.310677          | 0.311811            |</pre>"
      ],
      "text/plain": [
       "| 20        | 4.621799     | 0.854652          | 0.852696            | 0.310677          | 0.311811            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 25        | 5.673414     | 0.856222          | 0.854427            | 0.303004          | 0.304660            |</pre>"
      ],
      "text/plain": [
       "| 25        | 5.673414     | 0.856222          | 0.854427            | 0.303004          | 0.304660            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 30        | 6.689984     | 0.857622          | 0.855688            | 0.296405          | 0.298304            |</pre>"
      ],
      "text/plain": [
       "| 30        | 6.689984     | 0.857622          | 0.855688            | 0.296405          | 0.298304            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 35        | 7.724357     | 0.861943          | 0.859495            | 0.287702          | 0.289911            |</pre>"
      ],
      "text/plain": [
       "| 35        | 7.724357     | 0.861943          | 0.859495            | 0.287702          | 0.289911            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 40        | 8.781549     | 0.863912          | 0.862079            | 0.282889          | 0.285351            |</pre>"
      ],
      "text/plain": [
       "| 40        | 8.781549     | 0.863912          | 0.862079            | 0.282889          | 0.285351            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 45        | 9.837716     | 0.866845          | 0.864441            | 0.277284          | 0.280167            |</pre>"
      ],
      "text/plain": [
       "| 45        | 9.837716     | 0.866845          | 0.864441            | 0.277284          | 0.280167            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 50        | 10.850924    | 0.868022          | 0.865120            | 0.274630          | 0.277831            |</pre>"
      ],
      "text/plain": [
       "| 50        | 10.850924    | 0.868022          | 0.865120            | 0.274630          | 0.277831            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 51        | 11.065982    | 0.868551          | 0.865368            | 0.273235          | 0.276447            |</pre>"
      ],
      "text/plain": [
       "| 51        | 11.065982    | 0.868551          | 0.865368            | 0.273235          | 0.276447            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 55        | 11.886767    | 0.870671          | 0.867037            | 0.271037          | 0.274553            |</pre>"
      ],
      "text/plain": [
       "| 55        | 11.886767    | 0.870671          | 0.867037            | 0.271037          | 0.274553            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 60        | 12.963107    | 0.872136          | 0.868347            | 0.268328          | 0.272255            |</pre>"
      ],
      "text/plain": [
       "| 60        | 12.963107    | 0.872136          | 0.868347            | 0.268328          | 0.272255            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 65        | 14.015493    | 0.875425          | 0.871228            | 0.262729          | 0.267079            |</pre>"
      ],
      "text/plain": [
       "| 65        | 14.015493    | 0.875425          | 0.871228            | 0.262729          | 0.267079            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 70        | 15.135126    | 0.877563          | 0.872810            | 0.259009          | 0.263823            |</pre>"
      ],
      "text/plain": [
       "| 70        | 15.135126    | 0.877563          | 0.872810            | 0.259009          | 0.263823            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 75        | 16.222910    | 0.879430          | 0.874380            | 0.255404          | 0.260625            |</pre>"
      ],
      "text/plain": [
       "| 75        | 16.222910    | 0.879430          | 0.874380            | 0.255404          | 0.260625            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 80        | 17.340608    | 0.879705          | 0.874467            | 0.254240          | 0.259765            |</pre>"
      ],
      "text/plain": [
       "| 80        | 17.340608    | 0.879705          | 0.874467            | 0.254240          | 0.259765            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 85        | 18.522623    | 0.880679          | 0.875332            | 0.252314          | 0.258237            |</pre>"
      ],
      "text/plain": [
       "| 85        | 18.522623    | 0.880679          | 0.875332            | 0.252314          | 0.258237            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 90        | 19.552016    | 0.881516          | 0.875975            | 0.251165          | 0.257357            |</pre>"
      ],
      "text/plain": [
       "| 90        | 19.552016    | 0.881516          | 0.875975            | 0.251165          | 0.257357            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 95        | 20.627787    | 0.883263          | 0.877570            | 0.248907          | 0.255661            |</pre>"
      ],
      "text/plain": [
       "| 95        | 20.627787    | 0.883263          | 0.877570            | 0.248907          | 0.255661            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 100       | 21.718039    | 0.883785          | 0.877867            | 0.246837          | 0.253904            |</pre>"
      ],
      "text/plain": [
       "| 100       | 21.718039    | 0.883785          | 0.877867            | 0.246837          | 0.253904            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre>Boosted trees classifier:</pre>"
      ],
      "text/plain": [
       "Boosted trees classifier:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>--------------------------------------------------------</pre>"
      ],
      "text/plain": [
       "--------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of examples          : 323547</pre>"
      ],
      "text/plain": [
       "Number of examples          : 323547"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of classes           : 2</pre>"
      ],
      "text/plain": [
       "Number of classes           : 2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of feature columns   : 13</pre>"
      ],
      "text/plain": [
       "Number of feature columns   : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of unpacked features : 13</pre>"
      ],
      "text/plain": [
       "Number of unpacked features : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |</pre>"
      ],
      "text/plain": [
       "| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1         | 0.241397     | 0.744022          | 0.743407            | 0.587158          | 0.587395            |</pre>"
      ],
      "text/plain": [
       "| 1         | 0.241397     | 0.744022          | 0.743407            | 0.587158          | 0.587395            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2         | 0.458769     | 0.761114          | 0.761445            | 0.520424          | 0.520736            |</pre>"
      ],
      "text/plain": [
       "| 2         | 0.458769     | 0.761114          | 0.761445            | 0.520424          | 0.520736            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 3         | 0.733577     | 0.777439          | 0.778432            | 0.483173          | 0.483592            |</pre>"
      ],
      "text/plain": [
       "| 3         | 0.733577     | 0.777439          | 0.778432            | 0.483173          | 0.483592            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 4         | 0.972205     | 0.784538          | 0.784391            | 0.455668          | 0.456403            |</pre>"
      ],
      "text/plain": [
       "| 4         | 0.972205     | 0.784538          | 0.784391            | 0.455668          | 0.456403            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 5         | 1.226995     | 0.806050          | 0.803590            | 0.421505          | 0.422297            |</pre>"
      ],
      "text/plain": [
       "| 5         | 1.226995     | 0.806050          | 0.803590            | 0.421505          | 0.422297            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 6         | 1.467579     | 0.806285          | 0.803640            | 0.403970          | 0.404669            |</pre>"
      ],
      "text/plain": [
       "| 6         | 1.467579     | 0.806285          | 0.803640            | 0.403970          | 0.404669            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 10        | 2.498571     | 0.831959          | 0.829948            | 0.357311          | 0.358380            |</pre>"
      ],
      "text/plain": [
       "| 10        | 2.498571     | 0.831959          | 0.829948            | 0.357311          | 0.358380            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 11        | 2.738406     | 0.839152          | 0.837662            | 0.345970          | 0.347052            |</pre>"
      ],
      "text/plain": [
       "| 11        | 2.738406     | 0.839152          | 0.837662            | 0.345970          | 0.347052            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 15        | 3.570721     | 0.850288          | 0.849642            | 0.327286          | 0.328572            |</pre>"
      ],
      "text/plain": [
       "| 15        | 3.570721     | 0.850288          | 0.849642            | 0.327286          | 0.328572            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 20        | 4.635217     | 0.854924          | 0.853363            | 0.311074          | 0.312742            |</pre>"
      ],
      "text/plain": [
       "| 20        | 4.635217     | 0.854924          | 0.853363            | 0.311074          | 0.312742            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 25        | 5.681928     | 0.856695          | 0.854389            | 0.303361          | 0.305451            |</pre>"
      ],
      "text/plain": [
       "| 25        | 5.681928     | 0.856695          | 0.854389            | 0.303361          | 0.305451            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 30        | 6.726251     | 0.860033          | 0.857443            | 0.292450          | 0.295010            |</pre>"
      ],
      "text/plain": [
       "| 30        | 6.726251     | 0.860033          | 0.857443            | 0.292450          | 0.295010            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 35        | 7.801455     | 0.862144          | 0.859817            | 0.286051          | 0.288811            |</pre>"
      ],
      "text/plain": [
       "| 35        | 7.801455     | 0.862144          | 0.859817            | 0.286051          | 0.288811            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 40        | 8.951792     | 0.866628          | 0.864614            | 0.278924          | 0.281842            |</pre>"
      ],
      "text/plain": [
       "| 40        | 8.951792     | 0.866628          | 0.864614            | 0.278924          | 0.281842            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 45        | 10.040231    | 0.870161          | 0.868125            | 0.271560          | 0.274884            |</pre>"
      ],
      "text/plain": [
       "| 45        | 10.040231    | 0.870161          | 0.868125            | 0.271560          | 0.274884            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 50        | 11.178093    | 0.872408          | 0.869806            | 0.268721          | 0.272514            |</pre>"
      ],
      "text/plain": [
       "| 50        | 11.178093    | 0.872408          | 0.869806            | 0.268721          | 0.272514            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 51        | 11.411084    | 0.872609          | 0.869843            | 0.268380          | 0.272193            |</pre>"
      ],
      "text/plain": [
       "| 51        | 11.411084    | 0.872609          | 0.869843            | 0.268380          | 0.272193            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 55        | 12.290095    | 0.873910          | 0.871252            | 0.264418          | 0.268355            |</pre>"
      ],
      "text/plain": [
       "| 55        | 12.290095    | 0.873910          | 0.871252            | 0.264418          | 0.268355            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 60        | 13.424203    | 0.876738          | 0.873020            | 0.259878          | 0.264434            |</pre>"
      ],
      "text/plain": [
       "| 60        | 13.424203    | 0.876738          | 0.873020            | 0.259878          | 0.264434            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 65        | 14.551058    | 0.879220          | 0.875580            | 0.255756          | 0.260690            |</pre>"
      ],
      "text/plain": [
       "| 65        | 14.551058    | 0.879220          | 0.875580            | 0.255756          | 0.260690            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 70        | 15.687623    | 0.879575          | 0.875913            | 0.255216          | 0.260312            |</pre>"
      ],
      "text/plain": [
       "| 70        | 15.687623    | 0.879575          | 0.875913            | 0.255216          | 0.260312            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 75        | 16.796168    | 0.881566          | 0.877150            | 0.252218          | 0.257735            |</pre>"
      ],
      "text/plain": [
       "| 75        | 16.796168    | 0.881566          | 0.877150            | 0.252218          | 0.257735            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 80        | 17.954454    | 0.882969          | 0.878114            | 0.249976          | 0.255851            |</pre>"
      ],
      "text/plain": [
       "| 80        | 17.954454    | 0.882969          | 0.878114            | 0.249976          | 0.255851            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 85        | 19.042521    | 0.883584          | 0.878510            | 0.248992          | 0.255170            |</pre>"
      ],
      "text/plain": [
       "| 85        | 19.042521    | 0.883584          | 0.878510            | 0.248992          | 0.255170            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 90        | 20.246890    | 0.884660          | 0.879177            | 0.246051          | 0.252799            |</pre>"
      ],
      "text/plain": [
       "| 90        | 20.246890    | 0.884660          | 0.879177            | 0.246051          | 0.252799            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 95        | 21.360878    | 0.886502          | 0.880970            | 0.243663          | 0.250890            |</pre>"
      ],
      "text/plain": [
       "| 95        | 21.360878    | 0.886502          | 0.880970            | 0.243663          | 0.250890            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 100       | 22.426726    | 0.886959          | 0.881563            | 0.242856          | 0.250360            |</pre>"
      ],
      "text/plain": [
       "| 100       | 22.426726    | 0.886959          | 0.881563            | 0.242856          | 0.250360            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre>Boosted trees classifier:</pre>"
      ],
      "text/plain": [
       "Boosted trees classifier:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>--------------------------------------------------------</pre>"
      ],
      "text/plain": [
       "--------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of examples          : 323548</pre>"
      ],
      "text/plain": [
       "Number of examples          : 323548"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of classes           : 2</pre>"
      ],
      "text/plain": [
       "Number of classes           : 2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of feature columns   : 13</pre>"
      ],
      "text/plain": [
       "Number of feature columns   : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Number of unpacked features : 13</pre>"
      ],
      "text/plain": [
       "Number of unpacked features : 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |</pre>"
      ],
      "text/plain": [
       "| Iteration | Elapsed Time | Training-accuracy | Validation-accuracy | Training-log_loss | Validation-log_loss |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1         | 0.258009     | 0.743887          | 0.743305            | 0.587105          | 0.587466            |</pre>"
      ],
      "text/plain": [
       "| 1         | 0.258009     | 0.743887          | 0.743305            | 0.587105          | 0.587466            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2         | 0.467701     | 0.761148          | 0.760663            | 0.520225          | 0.520995            |</pre>"
      ],
      "text/plain": [
       "| 2         | 0.467701     | 0.761148          | 0.760663            | 0.520225          | 0.520995            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 3         | 0.719552     | 0.778673          | 0.778194            | 0.476859          | 0.477860            |</pre>"
      ],
      "text/plain": [
       "| 3         | 0.719552     | 0.778673          | 0.778194            | 0.476859          | 0.477860            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 4         | 0.950031     | 0.785343          | 0.785192            | 0.448986          | 0.450074            |</pre>"
      ],
      "text/plain": [
       "| 4         | 0.950031     | 0.785343          | 0.785192            | 0.448986          | 0.450074            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 5         | 1.196346     | 0.806625          | 0.806444            | 0.416947          | 0.417636            |</pre>"
      ],
      "text/plain": [
       "| 5         | 1.196346     | 0.806625          | 0.806444            | 0.416947          | 0.417636            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 6         | 1.438345     | 0.806968          | 0.807037            | 0.400704          | 0.401267            |</pre>"
      ],
      "text/plain": [
       "| 6         | 1.438345     | 0.806968          | 0.807037            | 0.400704          | 0.401267            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 10        | 2.357903     | 0.832387          | 0.833148            | 0.352671          | 0.353145            |</pre>"
      ],
      "text/plain": [
       "| 10        | 2.357903     | 0.832387          | 0.833148            | 0.352671          | 0.353145            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 11        | 2.574704     | 0.837020          | 0.837116            | 0.342360          | 0.342831            |</pre>"
      ],
      "text/plain": [
       "| 11        | 2.574704     | 0.837020          | 0.837116            | 0.342360          | 0.342831            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 15        | 3.501302     | 0.846150          | 0.844720            | 0.326503          | 0.327515            |</pre>"
      ],
      "text/plain": [
       "| 15        | 3.501302     | 0.846150          | 0.844720            | 0.326503          | 0.327515            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 20        | 4.614283     | 0.854093          | 0.851643            | 0.311831          | 0.313447            |</pre>"
      ],
      "text/plain": [
       "| 20        | 4.614283     | 0.854093          | 0.851643            | 0.311831          | 0.313447            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 25        | 5.735072     | 0.856108          | 0.853114            | 0.305561          | 0.307398            |</pre>"
      ],
      "text/plain": [
       "| 25        | 5.735072     | 0.856108          | 0.853114            | 0.305561          | 0.307398            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 30        | 6.905710     | 0.862694          | 0.860396            | 0.290544          | 0.293102            |</pre>"
      ],
      "text/plain": [
       "| 30        | 6.905710     | 0.862694          | 0.860396            | 0.290544          | 0.293102            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 35        | 8.097228     | 0.866273          | 0.863536            | 0.282386          | 0.285239            |</pre>"
      ],
      "text/plain": [
       "| 35        | 8.097228     | 0.866273          | 0.863536            | 0.282386          | 0.285239            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 40        | 9.245424     | 0.868023          | 0.864538            | 0.276991          | 0.280354            |</pre>"
      ],
      "text/plain": [
       "| 40        | 9.245424     | 0.868023          | 0.864538            | 0.276991          | 0.280354            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 45        | 10.459477    | 0.869364          | 0.865749            | 0.272197          | 0.275932            |</pre>"
      ],
      "text/plain": [
       "| 45        | 10.459477    | 0.869364          | 0.865749            | 0.272197          | 0.275932            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 50        | 11.561501    | 0.873450          | 0.868754            | 0.266220          | 0.270779            |</pre>"
      ],
      "text/plain": [
       "| 50        | 11.561501    | 0.873450          | 0.868754            | 0.266220          | 0.270779            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 51        | 11.773569    | 0.873673          | 0.868766            | 0.265966          | 0.270635            |</pre>"
      ],
      "text/plain": [
       "| 51        | 11.773569    | 0.873673          | 0.868766            | 0.265966          | 0.270635            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 55        | 12.627126    | 0.874111          | 0.869100            | 0.264857          | 0.269729            |</pre>"
      ],
      "text/plain": [
       "| 55        | 12.627126    | 0.874111          | 0.869100            | 0.264857          | 0.269729            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 60        | 13.729658    | 0.874844          | 0.869582            | 0.262300          | 0.267540            |</pre>"
      ],
      "text/plain": [
       "| 60        | 13.729658    | 0.874844          | 0.869582            | 0.262300          | 0.267540            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 65        | 14.835300    | 0.876510          | 0.871263            | 0.259838          | 0.265611            |</pre>"
      ],
      "text/plain": [
       "| 65        | 14.835300    | 0.876510          | 0.871263            | 0.259838          | 0.265611            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 70        | 15.922029    | 0.878281          | 0.872401            | 0.256340          | 0.262720            |</pre>"
      ],
      "text/plain": [
       "| 70        | 15.922029    | 0.878281          | 0.872401            | 0.256340          | 0.262720            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 75        | 16.991179    | 0.879536          | 0.873316            | 0.254625          | 0.261288            |</pre>"
      ],
      "text/plain": [
       "| 75        | 16.991179    | 0.879536          | 0.873316            | 0.254625          | 0.261288            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 80        | 18.077198    | 0.880858          | 0.874119            | 0.252315          | 0.259501            |</pre>"
      ],
      "text/plain": [
       "| 80        | 18.077198    | 0.880858          | 0.874119            | 0.252315          | 0.259501            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 85        | 19.206449    | 0.882524          | 0.875306            | 0.249966          | 0.257688            |</pre>"
      ],
      "text/plain": [
       "| 85        | 19.206449    | 0.882524          | 0.875306            | 0.249966          | 0.257688            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 90        | 20.287368    | 0.884963          | 0.876864            | 0.246291          | 0.254748            |</pre>"
      ],
      "text/plain": [
       "| 90        | 20.287368    | 0.884963          | 0.876864            | 0.246291          | 0.254748            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 95        | 21.370096    | 0.886119          | 0.877692            | 0.244133          | 0.253002            |</pre>"
      ],
      "text/plain": [
       "| 95        | 21.370096    | 0.886119          | 0.877692            | 0.244133          | 0.253002            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 100       | 22.415730    | 0.887185          | 0.878792            | 0.241564          | 0.251105            |</pre>"
      ],
      "text/plain": [
       "| 100       | 22.415730    | 0.887185          | 0.878792            | 0.241564          | 0.251105            |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+--------------+-------------------+---------------------+-------------------+---------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+--------------+-------------------+---------------------+-------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 4\n",
      "0.761380692026\n",
      "0.821533516399\n",
      "+--------------+-----------------+--------+\n",
      "| target_label | predicted_label | count  |\n",
      "+--------------+-----------------+--------+\n",
      "|      0       |        1        | 22251  |\n",
      "|      0       |        0        | 121829 |\n",
      "|      1       |        1        | 134266 |\n",
      "|      1       |        0        | 33382  |\n",
      "+--------------+-----------------+--------+\n",
      "[4 rows x 3 columns]\n",
      "\n",
      "0.828380608641\n",
      "6.16401397108\n",
      "0.857836528939\n",
      "0.80088041611\n",
      "+-----------+----------------+---------------+--------+--------+\n",
      "| threshold |      fpr       |      tpr      |   p    |   n    |\n",
      "+-----------+----------------+---------------+--------+--------+\n",
      "|    0.0    |      1.0       |      1.0      | 167648 | 144080 |\n",
      "|   1e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   2e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   3e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   4e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   5e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   6e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   7e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   8e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   9e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "+-----------+----------------+---------------+--------+--------+\n",
      "[100001 rows x 5 columns]\n",
      "Note: Only the head of the SFrame is printed.\n",
      "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n",
      "+-----------+----------------+---------------+--------+--------+\n",
      "| threshold |      fpr       |      tpr      |   p    |   n    |\n",
      "+-----------+----------------+---------------+--------+--------+\n",
      "|    0.0    |      1.0       |      1.0      | 167648 | 144080 |\n",
      "|   1e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   2e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   3e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   4e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   5e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   6e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   7e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   8e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "|   9e-05   | 0.154435036091 | 0.80088041611 | 167648 | 144080 |\n",
      "+-----------+----------------+---------------+--------+--------+\n",
      "[100001 rows x 5 columns]\n",
      "Note: Only the head of the SFrame is printed.\n",
      "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n"
     ]
    }
   ],
   "source": [
    "SF = gl.cross_validation.shuffle(SF)\n",
    "Folds = gl.cross_validation.KFold(SF,5)\n",
    "j=0\n",
    "for train,val in Folds:\n",
    "    model = gl.boosted_trees_classifier.create(train,features=myfeatures,target='isBot',validation_set=val,verbose = True,max_iterations=100)\n",
    "    models.append(model)\n",
    "    print \"Done\",j\n",
    "    j=j+1\n",
    "    \n",
    "pred = []\n",
    "for model in models:\n",
    "    temp = list(model.predict(Test))\n",
    "    pred.append(temp)\n",
    "    \n",
    "final = []\n",
    "for i in range(len(Test)):\n",
    "    count = 0\n",
    "    for j in range(5):\n",
    "        if pred[j][i] ==1:\n",
    "            count += 1\n",
    "    if count>2:\n",
    "        final.append(1)\n",
    "    else:\n",
    "        final.append(0)\n",
    "        \n",
    "print gl.toolkits.evaluation.auc(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.accuracy(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.confusion_matrix(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.f1_score(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.log_loss(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.precision(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.recall(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.roc_curve(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.roc_curve(Test['isBot'],gl.SArray(final))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 4\n",
      "Done 5\n",
      "Done 6\n",
      "Done 7\n",
      "Done 8\n",
      "0.610836556598\n",
      "0.636901401222\n",
      "+--------------+-----------------+--------+\n",
      "| target_label | predicted_label | count  |\n",
      "+--------------+-----------------+--------+\n",
      "|      0       |        1        | 24344  |\n",
      "|      0       |        0        | 119736 |\n",
      "|      1       |        1        | 78804  |\n",
      "|      1       |        0        | 88844  |\n",
      "+--------------+-----------------+--------+\n",
      "[4 rows x 3 columns]\n",
      "\n",
      "0.582017459638\n",
      "12.5409813125\n",
      "0.763989607166\n",
      "0.470056308456\n",
      "+-----------+----------------+----------------+--------+--------+\n",
      "| threshold |      fpr       |      tpr       |   p    |   n    |\n",
      "+-----------+----------------+----------------+--------+--------+\n",
      "|    0.0    |      1.0       |      1.0       | 167648 | 144080 |\n",
      "|   1e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   2e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   3e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   4e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   5e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   6e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   7e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   8e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   9e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "+-----------+----------------+----------------+--------+--------+\n",
      "[100001 rows x 5 columns]\n",
      "Note: Only the head of the SFrame is printed.\n",
      "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n",
      "+-----------+----------------+----------------+--------+--------+\n",
      "| threshold |      fpr       |      tpr       |   p    |   n    |\n",
      "+-----------+----------------+----------------+--------+--------+\n",
      "|    0.0    |      1.0       |      1.0       | 167648 | 144080 |\n",
      "|   1e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   2e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   3e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   4e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   5e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   6e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   7e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   8e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "|   9e-05   | 0.168961687951 | 0.470056308456 | 167648 | 144080 |\n",
      "+-----------+----------------+----------------+--------+--------+\n",
      "[100001 rows x 5 columns]\n",
      "Note: Only the head of the SFrame is printed.\n",
      "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n"
     ]
    }
   ],
   "source": [
    "models = []\n",
    "    \n",
    "#LOGISTIC REGRESSION\n",
    "j=1\n",
    "for train,val in Folds:\n",
    "    model = gl.logistic_classifier.create(train,features=myfeatures,target='isBot',validation_set=val,verbose = False,max_iterations=100)\n",
    "    models.append(model)\n",
    "    print \"Done\",j\n",
    "    j=j+1\n",
    "    \n",
    "pred = []\n",
    "for model in models:\n",
    "    temp = list(model.predict(Test))\n",
    "    pred.append(temp)\n",
    "    \n",
    "final = []\n",
    "for i in range(len(Test)):\n",
    "    count = 0\n",
    "    for j in range(5):\n",
    "        if pred[j][i] ==1:\n",
    "            count += 1\n",
    "    if count>2:\n",
    "        final.append(1)\n",
    "    else:\n",
    "        final.append(0)\n",
    "        \n",
    "print gl.toolkits.evaluation.auc(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.accuracy(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.confusion_matrix(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.f1_score(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.log_loss(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.precision(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.recall(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.roc_curve(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.roc_curve(Test['isBot'],gl.SArray(final))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done 0\n",
      "Done 1\n",
      "Done 2\n",
      "Done 3\n",
      "Done 4\n",
      "0.66963854541\n",
      "0.761840450649\n",
      "+--------------+-----------------+--------+\n",
      "| target_label | predicted_label | count  |\n",
      "+--------------+-----------------+--------+\n",
      "|      0       |        1        | 34699  |\n",
      "|      0       |        0        | 109381 |\n",
      "|      1       |        1        | 128106 |\n",
      "|      1       |        0        | 39542  |\n",
      "+--------------+-----------------+--------+\n",
      "[4 rows x 3 columns]\n",
      "\n",
      "0.775335675573\n",
      "8.22573942134\n",
      "0.786867725193\n",
      "0.764136762741\n",
      "+-----------+---------------+----------------+--------+--------+\n",
      "| threshold |      fpr      |      tpr       |   p    |   n    |\n",
      "+-----------+---------------+----------------+--------+--------+\n",
      "|    0.0    |      1.0      |      1.0       | 167648 | 144080 |\n",
      "|   1e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   2e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   3e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   4e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   5e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   6e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   7e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   8e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   9e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "+-----------+---------------+----------------+--------+--------+\n",
      "[100001 rows x 5 columns]\n",
      "Note: Only the head of the SFrame is printed.\n",
      "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n",
      "+-----------+---------------+----------------+--------+--------+\n",
      "| threshold |      fpr      |      tpr       |   p    |   n    |\n",
      "+-----------+---------------+----------------+--------+--------+\n",
      "|    0.0    |      1.0      |      1.0       | 167648 | 144080 |\n",
      "|   1e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   2e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   3e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   4e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   5e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   6e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   7e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   8e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "|   9e-05   | 0.24083148251 | 0.764136762741 | 167648 | 144080 |\n",
      "+-----------+---------------+----------------+--------+--------+\n",
      "[100001 rows x 5 columns]\n",
      "Note: Only the head of the SFrame is printed.\n",
      "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n"
     ]
    }
   ],
   "source": [
    "models = []\n",
    "SF = gl.cross_validation.shuffle(SF)\n",
    "Folds = gl.cross_validation.KFold(SF,5)\n",
    "j=0\n",
    "for train,val in Folds:\n",
    "    model = gl.random_forest_classifier.create(train,features=myfeatures,target='isBot',validation_set=val,verbose = False,max_iterations=100)\n",
    "    models.append(model)\n",
    "    print \"Done\",j\n",
    "    j=j+1\n",
    "    \n",
    "pred = []\n",
    "for model in models:\n",
    "    temp = list(model.predict(Test))\n",
    "    pred.append(temp)\n",
    "    \n",
    "final = []\n",
    "for i in range(len(Test)):\n",
    "    count = 0\n",
    "    for j in range(5):\n",
    "        if pred[j][i] ==1:\n",
    "            count += 1\n",
    "    if count>2:\n",
    "        final.append(1)\n",
    "    else:\n",
    "        final.append(0)\n",
    "        \n",
    "print gl.toolkits.evaluation.auc(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.accuracy(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.confusion_matrix(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.f1_score(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.log_loss(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.precision(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.recall(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.roc_curve(Test['isBot'],gl.SArray(final))\n",
    "print gl.toolkits.evaluation.roc_curve(Test['isBot'],gl.SArray(final))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SF.save('FinalTrain.csv')\n",
    "Test.save('FinalTest.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['FlowNo.',\n",
       " 'Info',\n",
       " 'udp_Length',\n",
       " 'Destination',\n",
       " 'APL',\n",
       " 'AvgPktPerSec',\n",
       " 'Destination Port',\n",
       " 'Answer RRs',\n",
       " 'TCP Segment Len',\n",
       " 'Differentiated Services Field',\n",
       " 'IOPR',\n",
       " 'Source',\n",
       " 'Length',\n",
       " 'Time',\n",
       " 'IAT',\n",
       " 'reconnects',\n",
       " 'NumForward',\n",
       " 'tcp_Flags',\n",
       " 'Protocols in frame',\n",
       " 'No.',\n",
       " 'Protocol',\n",
       " 'BytesEx',\n",
       " 'isNull',\n",
       " 'BitsPerSec',\n",
       " 'isBot',\n",
       " 'IP_Flags',\n",
       " 'NumPackets',\n",
       " 'Source Port',\n",
       " 'StdDevLen',\n",
       " 'Next sequence number',\n",
       " 'SameLenPktRatio',\n",
       " 'FPL',\n",
       " 'Duration',\n",
       " 'Time to live',\n",
       " 'Sequence number',\n",
       " 'NPEx']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SF.column_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SF[SF['AnswerRRs']>1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "582429"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SF[SF['IOPR']==-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iplist = [\n",
    "    \"147.32.84.180\",\n",
    "    \"147.32.84.170\",\n",
    "    \"147.32.84.150\",\n",
    "    \"147.32.84.140\",\n",
    "    \"147.32.84.130\",\n",
    "    \"147.32.84.160\",\n",
    "    \"10.0.2.15\",\n",
    "    \"192.168.106.141\",\n",
    "    \"192.168.106.131\",\n",
    "    \"172.16.253.130\",\n",
    "    \"172.16.253.131\",\n",
    "    \"172.16.253.129\",\n",
    "    \"172.16.253.240\",\n",
    "    \"74.78.117.238\",\n",
    "    \"158.65.110.24\",\n",
    "    \"192.168.3.35\",\n",
    "    \"192.168.3.25\",\n",
    "    \"192.168.3.65\",\n",
    "    \"172.29.0.116\",\n",
    "    \"172.29.0.109\",\n",
    "    \"172.16.253.132\",\n",
    "    \"192.168.248.165\",\n",
    "    \"10.37.130.4\",\n",
    "    \"131.202.243.84\", \n",
    "    \"198.164.30.2\", \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \n",
    "    \"192.168.5.122\", \"192.168.5.122\", \"192.168.5.122\", \n",
    "    \"192.168.2.110\", \n",
    "    \"192.168.4.120\", \"192.168.1.103\",\n",
    "    \"192.168.2.113\", \"192.168.4.118\" , \"192.168.2.109\", \"192.168.2.105\", \"192.168.5.122\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Test.add_column(gl.SArray(final),'Predicted_Label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "truePosIPs = Test[(Test['isBot']==1) & (Test['Predicted_Label']==1)]['Source'].unique()\n",
    "i=0\n",
    "for x in truePosIPs:\n",
    "    if x in iplist:\n",
    "        print x\n",
    "        i=i+1\n",
    "    else:\n",
    "        print \"Not in Malicious IP List \" + x\n",
    "print i\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "falseNegIPs = Test[(Test['isBot']==1) & (Test['Predicted_Label']==0)]['Source'].unique()\n",
    "len(falseNegIPs)\n",
    "\n",
    "i=0\n",
    "for x in falseNegIPs:\n",
    "    if x in iplist:\n",
    "        print x\n",
    "        i=i+1\n",
    "    else:\n",
    "        print \"Not in Malicious IP List \" + x\n",
    "print i\n",
    "\n",
    "BotsInTrain = SF[(SF['isBot']==1)]['Source'].unique()\n",
    "\n",
    "i=0\n",
    "for x in falseNegIPs:\n",
    "    if x in BotsInTrain:\n",
    "        print x + \" in both train and test, still not detected\"\n",
    "    else:\n",
    "        i=i+1\n",
    "print str(i) + \" Bot IPs present in test, not in train, undetected \"\n",
    "\n",
    "i=0\n",
    "for x in truePosIPs:\n",
    "    if x in BotsInTrain:\n",
    "        print x + \" in both train and test, detected\"\n",
    "    else:\n",
    "        i=i+1\n",
    "print str(i) + \" Bot IPs present in test, not in train, still detected => Novelty Detection \""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 22-10-2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[INFO] graphlab.cython.cy_server: GraphLab Create v2.1 started. Logging: /tmp/graphlab_server_1477113345.log\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This non-commercial license of GraphLab Create for academic use is assigned to himadri.mishra.cse13@iitbhu.ac.in and will expire on September 21, 2017.\n"
     ]
    }
   ],
   "source": [
    "import graphlab as gl\n",
    "import tensorflow as tf\n",
    "SF = gl.SFrame.read_csv('ISCX_Botnet-Testing_Ports_Only_Sorted_Flow_IAT_Null.csv',verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This non-commercial license of GraphLab Create for academic use is assigned to himadri.mishra.cse13@iitbhu.ac.in and will expire on September 21, 2017.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[INFO] graphlab.cython.cy_server: GraphLab Create v2.1 started. Logging: /tmp/graphlab_server_1477323264.log\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre>Finished parsing file /home/os-net/kartik-himadri-BTP/Dataset/ISCX_Botnet-Testing.csv</pre>"
      ],
      "text/plain": [
       "Finished parsing file /home/os-net/kartik-himadri-BTP/Dataset/ISCX_Botnet-Testing.csv"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Parsing completed. Parsed 100 lines in 0.986556 secs.</pre>"
      ],
      "text/plain": [
       "Parsing completed. Parsed 100 lines in 0.986556 secs."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "Inferred types from first 100 line(s) of file as \n",
      "column_type_hints=[int,float,str,str,str,int,str,str,str,str,str,str,str,str,str,str,str,str,str,str,str,str,str,str,str]\n",
      "If parsing fails due to incorrect types, you can correct\n",
      "the inferred type list above and pass it to read_csv in\n",
      "the column_type_hints argument\n",
      "------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre>Read 176514 lines. Lines per second: 122280</pre>"
      ],
      "text/plain": [
       "Read 176514 lines. Lines per second: 122280"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Read 1471970 lines. Lines per second: 212693</pre>"
      ],
      "text/plain": [
       "Read 1471970 lines. Lines per second: 212693"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Read 2878577 lines. Lines per second: 225595</pre>"
      ],
      "text/plain": [
       "Read 2878577 lines. Lines per second: 225595"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Read 4058710 lines. Lines per second: 227091</pre>"
      ],
      "text/plain": [
       "Read 4058710 lines. Lines per second: 227091"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Finished parsing file /home/os-net/kartik-himadri-BTP/Dataset/ISCX_Botnet-Testing.csv</pre>"
      ],
      "text/plain": [
       "Finished parsing file /home/os-net/kartik-himadri-BTP/Dataset/ISCX_Botnet-Testing.csv"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Parsing completed. Parsed 5114514 lines in 22.441 secs.</pre>"
      ],
      "text/plain": [
       "Parsing completed. Parsed 5114514 lines in 22.441 secs."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import graphlab as gl\n",
    "tp = gl.SFrame.read_csv('ISCX_Botnet-Testing.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0x0700 4684\n",
      "0x3339 1\n",
      "0x4000 83\n",
      "0x4b0d 1\n",
      "0xe000 1\n",
      "0xffff 2773\n",
      "ARP 188815\n",
      "Ethernet 1\n",
      "ICMP 7423\n",
      "ICMPv6 1686\n",
      "IGMPv1 4\n",
      "IGMPv2 30\n",
      "IGMPv3 1010\n",
      "IPv4 140010\n",
      "IPv6 48\n",
      "LLC 140\n",
      "LLDP 5684\n",
      "STP 85272\n",
      "TDMoP 529\n"
     ]
    }
   ],
   "source": [
    "tp = tp[(tp['Source Port']=='')|(tp['Destination Port']=='')]\n",
    "for x in sorted(tp['Protocol'].unique()):\n",
    "    print x,len(tp[tp['Protocol']==x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0x0700 4684\n",
      "0x3339 1\n",
      "0x4000 83\n",
      "0x4b0d 1\n",
      "0xe000 1\n",
      "0xffff 2773\n",
      "ARP 188815\n",
      "Ethernet 1\n",
      "ICMP 7423\n",
      "ICMPv6 1686\n",
      "IGMPv1 4\n",
      "IGMPv2 30\n",
      "IGMPv3 1010\n",
      "IPv4 140010\n",
      "IPv6 48\n",
      "LLC 140\n",
      "LLDP 5684\n",
      "STP 85272\n",
      "TDMoP 529\n"
     ]
    }
   ],
   "source": [
    "tp = tp[(tp['Source Port']=='')]\n",
    "for x in sorted(tp['Protocol'].unique()):\n",
    "    print x,len(tp[tp['Protocol']==x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0x0700 4684\n",
      "0x3339 1\n",
      "0x4000 83\n",
      "0x4b0d 1\n",
      "0xe000 1\n",
      "0xffff 2773\n",
      "ARP 188815\n",
      "Ethernet 1\n",
      "ICMP 7423\n",
      "ICMPv6 1686\n",
      "IGMPv1 4\n",
      "IGMPv2 30\n",
      "IGMPv3 1010\n",
      "IPv4 140010\n",
      "IPv6 48\n",
      "LLC 140\n",
      "LLDP 5684\n",
      "STP 85272\n",
      "TDMoP 529\n"
     ]
    }
   ],
   "source": [
    "tp = tp[(tp['Destination Port']=='')]\n",
    "for x in sorted(tp['Protocol'].unique()):\n",
    "    print x,len(tp[tp['Protocol']==x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "438195"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4684 , 1 , 83 , 1 , 1 , 2773 , 188815 , 1 , 7423 , 1686 , 4 , 30 , 1010 , 140010 , 48 , 140 , 5684 , 85272 , 529 ,\n"
     ]
    }
   ],
   "source": [
    "for x in sorted(tp['Protocol'].unique()):\n",
    "    print len(tp[tp['Protocol']==x]),',',"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}