visualisation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import time\n",
    "import json\n",
    "import numpy as np\n",
    "import math\n",
    "import random\n",
    "import xml.etree.ElementTree as ET\n",
    "from subprocess import check_output\n",
    "from subprocess import CalledProcessError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x10ce83cb0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1337)\n",
    "random.seed(1337)\n",
    "torch.manual_seed(1337)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(torch.nn.Module):\n",
    "    def __init__(self, gen_emb, domain_emb, num_classes=3, dropout=0.5, crf=False):\n",
    "        super(Model, self).__init__()\n",
    "        self.gen_embedding = torch.nn.Embedding(gen_emb.shape[0], gen_emb.shape[1])\n",
    "        self.gen_embedding.weight=torch.nn.Parameter(torch.from_numpy(gen_emb), requires_grad=False)\n",
    "        self.domain_embedding = torch.nn.Embedding(domain_emb.shape[0], domain_emb.shape[1])\n",
    "        self.domain_embedding.weight=torch.nn.Parameter(torch.from_numpy(domain_emb), requires_grad=False)\n",
    "    \n",
    "        self.conv1=torch.nn.Conv1d(gen_emb.shape[1]+domain_emb.shape[1], 128, 5, padding=2 )\n",
    "        self.conv2=torch.nn.Conv1d(gen_emb.shape[1]+domain_emb.shape[1], 128, 3, padding=1 )\n",
    "        self.dropout=torch.nn.Dropout(dropout)\n",
    "\n",
    "        self.conv3=torch.nn.Conv1d(256, 256, 5, padding=2)\n",
    "        self.conv4=torch.nn.Conv1d(256, 256, 5, padding=2)\n",
    "        self.conv5=torch.nn.Conv1d(256, 256, 5, padding=2)\n",
    "        self.linear_ae=torch.nn.Linear(256, num_classes)\n",
    "        self.crf_flag=crf\n",
    "        if self.crf_flag:\n",
    "            from allennlp.modules import ConditionalRandomField\n",
    "            self.crf=ConditionalRandomField(num_classes)            \n",
    "          \n",
    "    def forward(self, x, x_len, x_mask, x_tag=None, testing=False):\n",
    "        x_emb=torch.cat((self.gen_embedding(x), self.domain_embedding(x) ), dim=2)\n",
    "        x_emb=self.dropout(x_emb).transpose(1, 2)\n",
    "        x_conv=torch.nn.functional.relu(torch.cat((self.conv1(x_emb), self.conv2(x_emb)), dim=1) )\n",
    "        x_conv=self.dropout(x_conv)\n",
    "        x_conv=torch.nn.functional.relu(self.conv3(x_conv) )\n",
    "        x_conv=self.dropout(x_conv)\n",
    "        x_conv=torch.nn.functional.relu(self.conv4(x_conv) )\n",
    "        x_conv=self.dropout(x_conv)\n",
    "        x_conv=torch.nn.functional.relu(self.conv5(x_conv) )\n",
    "        x_conv=x_conv.transpose(1, 2)\n",
    "        x_logit=self.linear_ae(x_conv)\n",
    "        if testing:\n",
    "            if self.crf_flag:\n",
    "                score=self.crf.viterbi_tags(x_logit, x_mask)\n",
    "            else:\n",
    "                x_logit=x_logit.transpose(2, 0)\n",
    "                score=torch.nn.functional.log_softmax(x_logit).transpose(2, 0)\n",
    "        else:\n",
    "            if self.crf_flag:\n",
    "                score=-self.crf(x_logit, x_tag, x_mask)\n",
    "            else:\n",
    "                x_logit=torch.nn.utils.rnn.pack_padded_sequence(x_logit, x_len, batch_first=True)\n",
    "                score=torch.nn.functional.nll_loss(torch.nn.functional.log_softmax(x_logit.data), x_tag.data)\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5\">But \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Google\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " is starting from behind.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc = nlp(u\"But Google is starting from behind.\")\n",
    "displacy.render(doc, style=\"ent\", jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    But\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Google\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ambience</span>\n",
       "</mark>\n",
       " is starting \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    from\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drinks</span>\n",
       "</mark>\n",
       " behind.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = [{\"text\": \"But Google is starting from behind.\",\n",
    "       \"ents\": [{\"start\":0, \"end\":3, \"label\":\"food\"}, {\"start\": 4, \"end\": 10, \"label\": \"ambience\"}, {\"start\": 23, \"end\": 27, \"label\": \"drinks\"} ],\n",
    "       \"title\": \"Review\"}]\n",
    "html = displacy.render(x, style=\"ent\", manual=True, jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read reviews\n",
    "# get BOI encoding\n",
    "# make html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "runs = 1\n",
    "data_dir = \"/sem8/COL772-NLP/DE-CNN/data/prep_data/\"\n",
    "model_dir = \"/sem8/COL772-NLP/DE-CNN/script/model/\"\n",
    "domain = \"restaurant\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "command=\"java --add-modules java.xml.bind -cp /sem8/COL772-NLP/DE-CNN/script/A.jar absa16.Do Eval -prd /sem8/COL772-NLP/DE-CNN/data/official_data/pred.xml -gld /sem8/COL772-NLP/DE-CNN/data/official_data/EN_REST_SB1_TEST.xml.gold -evs 2 -phs A -sbt SB1\"\n",
    "template=\"/sem8/COL772-NLP/DE-CNN/data/official_data/EN_REST_SB1_TEST.xml.A\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ae_data=np.load(data_dir+domain+\".npz\")\n",
    "with open(data_dir+domain+\"_raw_test.json\") as f:\n",
    "    raw_X=json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "r=0\n",
    "test_X = ae_data['test_X']\n",
    "batch_size=128\n",
    "crf = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/serialization.py:400: UserWarning: Couldn't retrieve source code for container of type Model. It won't be checked for correctness upon loading.\n",
      "  \"type \" + container_type.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "model=torch.load(model_dir+domain+str(r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (gen_embedding): Embedding(8518, 300)\n",
       "  (domain_embedding): Embedding(8518, 100)\n",
       "  (conv1): Conv1d(400, 128, kernel_size=(5,), stride=(1,), padding=(2,))\n",
       "  (conv2): Conv1d(400, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "  (dropout): Dropout(p=0.55)\n",
       "  (conv3): Conv1d(256, 256, kernel_size=(5,), stride=(1,), padding=(2,))\n",
       "  (conv4): Conv1d(256, 256, kernel_size=(5,), stride=(1,), padding=(2,))\n",
       "  (conv5): Conv1d(256, 256, kernel_size=(5,), stride=(1,), padding=(2,))\n",
       "  (linear_ae): Linear(in_features=256, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/navreetkaur/Library/Python/3.7/lib/python/site-packages/ipykernel_launcher.py:39: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 2 2 2]\n",
      " [1 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " ...\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]]\n",
      "[[0 1 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 1 2 ... 2 2 2]\n",
      " ...\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]]\n",
      "[[0 0 1 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " ...\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]]\n",
      "[[0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " ...\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]]\n",
      "[[0 1 2 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " ...\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 1 1 ... 2 2 2]]\n",
      "[[0 1 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]\n",
      " [0 1 0 ... 2 2 2]\n",
      " ...\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]]\n",
      "COMMAND: \n",
      "['java', '--add-modules', 'java.xml.bind', '-cp', '/sem8/COL772-NLP/DE-CNN/script/A.jar', 'absa16.Do', 'Eval', '-prd', '/sem8/COL772-NLP/DE-CNN/data/official_data/pred.xml', '-gld', '/sem8/COL772-NLP/DE-CNN/data/official_data/EN_REST_SB1_TEST.xml.gold', '-evs', '2', '-phs', 'A', '-sbt', 'SB1']\n",
      "[[0 0 0 ... 2 2 2]\n",
      " [1 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " ...\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]\n",
      " [0 0 0 ... 2 2 2]]\n"
     ]
    }
   ],
   "source": [
    "pred_y=np.zeros((test_X.shape[0], 83), np.int16)\n",
    "model.eval()\n",
    "for offset in range(0, test_X.shape[0], batch_size):\n",
    "    batch_test_X_len=np.sum(test_X[offset:offset+batch_size]!=0, axis=1)\n",
    "    batch_idx=batch_test_X_len.argsort()[::-1]\n",
    "    batch_test_X_len=batch_test_X_len[batch_idx]\n",
    "    batch_test_X_mask=(test_X[offset:offset+batch_size]!=0)[batch_idx].astype(np.uint8)\n",
    "    batch_test_X=test_X[offset:offset+batch_size][batch_idx]\n",
    "    # batch_test_X_mask=torch.autograd.Variable(torch.from_numpy(batch_test_X_mask).long().cuda() )\n",
    "    batch_test_X_mask=torch.autograd.Variable(torch.from_numpy(batch_test_X_mask).long())\n",
    "    # batch_test_X = torch.autograd.Variable(torch.from_numpy(batch_test_X).long().cuda() )\n",
    "    batch_test_X = torch.autograd.Variable(torch.from_numpy(batch_test_X).long())\n",
    "    batch_pred_y=model(batch_test_X, batch_test_X_len, batch_test_X_mask, testing=True)\n",
    "    r_idx=batch_idx.argsort()\n",
    "    if crf:\n",
    "        batch_pred_y=[batch_pred_y[idx] for idx in r_idx]\n",
    "        for ix in range(len(batch_pred_y) ):\n",
    "            for jx in range(len(batch_pred_y[ix]) ):\n",
    "                pred_y[offset+ix,jx]=batch_pred_y[ix][jx]\n",
    "    else:\n",
    "        batch_pred_y=batch_pred_y.data.cpu().numpy().argmax(axis=2)[r_idx]\n",
    "        pred_y[offset:offset+batch_size,:batch_pred_y.shape[1]]=batch_pred_y\n",
    "        print(batch_pred_y)\n",
    "model.train()\n",
    "assert len(pred_y)==len(test_X)\n",
    "\n",
    "command=command.split()\n",
    "print(\"COMMAND: \")\n",
    "print(command)\n",
    "if domain=='restaurant':\n",
    "    print(pred_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_rest_xml(fn, output_fn, corpus, label):\n",
    "    dom=ET.parse(fn)\n",
    "    root=dom.getroot()\n",
    "    pred_y=[]\n",
    "    for zx, sent in enumerate(root.iter(\"sentence\") ) :\n",
    "        tokens=corpus[zx]\n",
    "        lb=label[zx]\n",
    "        opins=ET.Element(\"Opinions\")\n",
    "        token_idx, pt, tag_on=0, 0, False\n",
    "        start, end=-1, -1\n",
    "        for ix, c in enumerate(sent.find('text').text):\n",
    "            if token_idx<len(tokens) and pt>=len(tokens[token_idx] ):\n",
    "                pt=0\n",
    "                token_idx+=1\n",
    "\n",
    "            if token_idx<len(tokens) and lb[token_idx]==1 and pt==0 and c!=' ':\n",
    "                if tag_on:\n",
    "                    end=ix\n",
    "                    tag_on=False\n",
    "                    opin=ET.Element(\"Opinion\")\n",
    "                    opin.attrib['target']=sent.find('text').text[start:end]\n",
    "                    opin.attrib['from']=str(start)\n",
    "                    opin.attrib['to']=str(end)\n",
    "                    opins.append(opin)\n",
    "                start=ix\n",
    "                tag_on=True\n",
    "            elif token_idx<len(tokens) and lb[token_idx]==2 and pt==0 and c!=' ' and not tag_on:\n",
    "                start=ix\n",
    "                tag_on=True\n",
    "            elif token_idx<len(tokens) and (lb[token_idx]==0 or lb[token_idx]==1) and tag_on and pt==0:\n",
    "                end=ix\n",
    "                tag_on=False \n",
    "                opin=ET.Element(\"Opinion\")\n",
    "                opin.attrib['target']=sent.find('text').text[start:end]\n",
    "                opin.attrib['from']=str(start)\n",
    "                opin.attrib['to']=str(end)\n",
    "                opins.append(opin)\n",
    "            elif token_idx>=len(tokens) and tag_on:\n",
    "                end=ix\n",
    "                tag_on=False \n",
    "                opin=ET.Element(\"Opinion\")\n",
    "                opin.attrib['target']=sent.find('text').text[start:end]\n",
    "                opin.attrib['from']=str(start)\n",
    "                opin.attrib['to']=str(end)\n",
    "                opins.append(opin)\n",
    "            if c==' ':\n",
    "                pass\n",
    "            elif tokens[token_idx][pt:pt+2]=='``' or tokens[token_idx][pt:pt+2]==\"''\":\n",
    "                pt+=2\n",
    "            else:\n",
    "                pt+=1\n",
    "        if tag_on:\n",
    "            tag_on=False\n",
    "            end=len(sent.find('text').text)\n",
    "            opin=ET.Element(\"Opinion\")\n",
    "            opin.attrib['target']=sent.find('text').text[start:end]\n",
    "            opin.attrib['from']=str(start)\n",
    "            opin.attrib['to']=str(end)\n",
    "            opins.append(opin)\n",
    "        sent.append(opins )\n",
    "    dom.write(output_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['Yum', '!'],\n",
       " array([0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], dtype=int16),\n",
       " array([7886,  123,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0], dtype=int32))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_X[0], pred_y[0], test_X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], dtype=int16)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_y[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = []\n",
    "labels = []\n",
    "previous_y = 0\n",
    "for i, sent in enumerate(raw_X):\n",
    "    text = \"\"\n",
    "    ents = []\n",
    "    start = 0\n",
    "    curr_start = 0\n",
    "    curr_end = 0\n",
    "    total_chars = 0\n",
    "    for j, word in enumerate(sent):\n",
    "        curr_num_char = len(word)\n",
    "        text+= word+\" \"\n",
    "        if pred_y[i][j] == 0:\n",
    "            if previous_y ==1:\n",
    "                curr_end = total_chars-1\n",
    "                ents.append((curr_start,curr_end, \" 1 \"))\n",
    "            elif previous_y ==2:\n",
    "                curr_end = total_chars-1\n",
    "                ents.append((curr_start,curr_end, \" 2 \"))\n",
    "        if pred_y[i][j] ==1:\n",
    "            curr_start = total_chars\n",
    "        if pred_y[i][j] ==2:\n",
    "            curr_end = total_chars+curr_num_char\n",
    "        total_chars += curr_num_char+1\n",
    "        previous_y=pred_y[i][j]\n",
    "    texts.append(text)\n",
    "    labels.append(ents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ents in labels:\n",
    "    e = []\n",
    "    for ent in ents:\n",
    "        d = {}\n",
    "        d['start'] = ent[0]\n",
    "        d['end'] = ent[1]\n",
    "        d['label'] = \"food\"\n",
    "        e.append(d)\n",
    "    ls.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = [{\"text\": \"But Google is starting from behind.\",\n",
    "       \"ents\": [{\"start\":0, \"end\":3, \"label\":\"food\"}, {\"start\": 4, \"end\": 10, \"label\": \"ambience\"}, {\"start\": 23, \"end\": 27, \"label\": \"drinks\"} ],\n",
    "       \"title\": \"Review\"}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_encoding = []\n",
    "for i in range(len(texts)):\n",
    "    text = texts[i]\n",
    "    ents = ls[i]\n",
    "    d = {}\n",
    "    d['text'] = text\n",
    "    d['ents'] = ents\n",
    "    d['title'] = \"Review\"\n",
    "    data_encoding.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': 'Yum ! ', 'ents': [], 'title': 'Review'},\n",
       " {'text': 'Serves really good sushi . ',\n",
       "  'ents': [{'start': 0, 'end': 6, 'label': 'food'},\n",
       "   {'start': 19, 'end': 24, 'label': 'food'}],\n",
       "  'title': 'Review'},\n",
       " {'text': 'Not the biggest portions but adequate . ',\n",
       "  'ents': [{'start': 16, 'end': 24, 'label': 'food'}],\n",
       "  'title': 'Review'},\n",
       " {'text': 'Green Tea creme brulee is a must ! ',\n",
       "  'ents': [{'start': 0, 'end': 22, 'label': 'food'}],\n",
       "  'title': 'Review'},\n",
       " {'text': \"Do n't leave the restaurant without it . \",\n",
       "  'ents': [],\n",
       "  'title': 'Review'},\n",
       " {'text': 'No Comparison ', 'ents': [], 'title': 'Review'},\n",
       " {'text': \"– I ca n't say enough about this place . \",\n",
       "  'ents': [{'start': 33, 'end': 38, 'label': 'food'}],\n",
       "  'title': 'Review'},\n",
       " {'text': 'It has great sushi and even better service . ',\n",
       "  'ents': [{'start': 13, 'end': 18, 'label': 'food'},\n",
       "   {'start': 35, 'end': 42, 'label': 'food'}],\n",
       "  'title': 'Review'},\n",
       " {'text': 'The entire staff was extremely accomodating and tended to my every need . ',\n",
       "  'ents': [{'start': 4, 'end': 16, 'label': 'food'}],\n",
       "  'title': 'Review'},\n",
       " {'text': \"I 've been to this restaurant over a dozen times with no complaints to date . \",\n",
       "  'ents': [{'start': 19, 'end': 29, 'label': 'food'}],\n",
       "  'title': 'Review'}]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_encoding[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">Yum ! </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Serves\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " really good \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    sushi\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">Not the biggest \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    portions\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " but adequate . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Green Tea creme brulee\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " is a must ! </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">Do n't leave the restaurant without it . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">No Comparison </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">– I ca n't say enough about this \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    place\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">It has great \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    sushi\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " and even better \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    service\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">The \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    entire staff\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " was extremely accomodating and tended to my every need . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2 style=\"margin: 0\">Review</h2>\n",
       "\n",
       "<div class=\"entities\" style=\"line-height: 2.5\">I 've been to this \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    restaurant\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">food</span>\n",
       "</mark>\n",
       " over a dozen times with no complaints to date . </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for x in data_encoding[:10]:\n",
    "    html = displacy.render(x, style=\"ent\", manual=True, jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0b3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}