Step 1-Data_preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "data_preprocessing.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y0ZQWJNLvS1v",
        "colab_type": "text"
      },
      "source": [
        "# **Data Preprocessing**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pt3LGJWbsJR9",
        "colab_type": "code",
        "outputId": "ecab7398-609f-4232-fd8c-53d096f883fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 100
        }
      },
      "source": [
        "import pandas as pd\n",
        "import nltk\n",
        "import gensim\n",
        "from gensim.utils import simple_preprocess\n",
        "from nltk.corpus import stopwords\n",
        "from nltk import word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
        "from nltk.stem.porter import *\n",
        "from gensim.parsing.preprocessing import STOPWORDS\n",
        "import numpy as np\n",
        "import pickle\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POGuQL62tSMc",
        "colab_type": "code",
        "outputId": "c2ef823d-d930-4dd7-b9e9-d6ada483633a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        }
      },
      "source": [
        "#mounting the google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VQe8RAJpsJSQ",
        "colab_type": "code",
        "outputId": "eba6ebaa-6879-4e5d-9234-1ec51e491333",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "#importing the CSV file of the dataset from google drive into a dataframe\n",
        "data=pd.read_csv('drive/My Drive/duplicate_detection/eclipse_platform.csv')\n",
        "data"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Issue_id</th>\n",
              "      <th>Priority</th>\n",
              "      <th>Component</th>\n",
              "      <th>Duplicated_issue</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>Status</th>\n",
              "      <th>Resolution</th>\n",
              "      <th>Version</th>\n",
              "      <th>Created_time</th>\n",
              "      <th>Resolved_time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>P3</td>\n",
              "      <td>Team</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Usability issue with external editors (1GE6IRL)</td>\n",
              "      <td>- Setup a project that contains a *.gif resource; \\t- release project to CVS; \\t- edit the *.gif resource with an external editor; e.g. PaintShop; \\t- save and close external editor; \\t- in Navigator open the icon resource and verify that your changes are there; \\t- release project; \\t\\t-&gt; nothing to release!; \\t- in Navigator open the icon resource and verify that your changes are still there; ; \\tProblem: because I never Refreshed from local; the workspace hasnt changed so Release didnt fi...</td>\n",
              "      <td>CLOSED</td>\n",
              "      <td>FIXED</td>\n",
              "      <td>2</td>\n",
              "      <td>2001-10-10 21:34</td>\n",
              "      <td>2012-02-09 15:57</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>P5</td>\n",
              "      <td>Team</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Opening repository resources doesnt honor type (1GE6I1Q)</td>\n",
              "      <td>Opening repository resource always open the default text editor and doesnt; \\thonor any mapping between resource types and editors.; \\tAs a result it is not possible to view the contents of an image (*.gif file) in a sensible way.; ; NOTES:; EG (5/24/2001 5:13:45 AM); \\tAgreed that this is needed eventually but not critical for June. \\t; \\tThere should also be a Java outliner when editing a java resource from the repository.; \\tMoving to VCM.; ; KM (5/24/01 2:09:21 PM); \\tIn general you cann...</td>\n",
              "      <td>RESOLVED</td>\n",
              "      <td>FIXED</td>\n",
              "      <td>2</td>\n",
              "      <td>2001-10-10 21:34</td>\n",
              "      <td>2002-05-07 10:33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>P5</td>\n",
              "      <td>Team</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Sync does not indicate deletion (1GIEN83)</td>\n",
              "      <td>KM (10/2/2001 5:55:18 PM); \\tThis PR about the deletion indicator in the sync viewer being too subtle.; \\tThis would be even more true for someone with vision problems.; \\tNE has good suggestion of using the label of the compare text viewer.; \\tWe may also consider a better icon.; \\tOn to the PR....; ; - delete file from project; - synchronize; - there is no indication that the file has been deleted from the workspace; - it looks the same as if the file is in the workspace but is empty; ; NO...</td>\n",
              "      <td>RESOLVED</td>\n",
              "      <td>FIXED</td>\n",
              "      <td>2</td>\n",
              "      <td>2001-10-10 21:34</td>\n",
              "      <td>2010-05-07 10:28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>P5</td>\n",
              "      <td>Team</td>\n",
              "      <td>NaN</td>\n",
              "      <td>need better error message if catching up over read-only resource (1GF69TF)</td>\n",
              "      <td>- become synchronized with some project in a repository; - use a different Eclipse to make a change to a file resource within this project and release it to the repository; - in the original Eclipse mark this file resource as being read-only (select it; right-click -&gt; Properties; change; OK); - select the file resource; right-click -&gt; Team -&gt; Synchronize with Stream; - in the subsequent comparison view select the file resource; right-click -&gt; Catchup; - since it has been marked as read-only ...</td>\n",
              "      <td>RESOLVED</td>\n",
              "      <td>FIXED</td>\n",
              "      <td>2</td>\n",
              "      <td>2001-10-10 21:34</td>\n",
              "      <td>2002-03-01 16:27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>P3</td>\n",
              "      <td>Team</td>\n",
              "      <td>NaN</td>\n",
              "      <td>ISharingManager sharing API inconsistent (1GAUL8H)</td>\n",
              "      <td>For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:</td>\n",
              "      <td>RESOLVED</td>\n",
              "      <td>WONTFIX</td>\n",
              "      <td>2</td>\n",
              "      <td>2001-10-10 21:34</td>\n",
              "      <td>2008-08-15 8:04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85151</th>\n",
              "      <td>424679</td>\n",
              "      <td>P3</td>\n",
              "      <td>IDE</td>\n",
              "      <td>390071.0</td>\n",
              "      <td>Crash on bootstarp</td>\n",
              "      <td>NaN</td>\n",
              "      <td>CLOSED</td>\n",
              "      <td>DUPLICATE</td>\n",
              "      <td>4.3</td>\n",
              "      <td>2013-12-26 20:12</td>\n",
              "      <td>2013-12-31 5:32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85152</th>\n",
              "      <td>424714</td>\n",
              "      <td>P3</td>\n",
              "      <td>UI</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[GTK/Linux] Blank Windows with GTK3</td>\n",
              "      <td>I started testing 4.4 and noticed the editor windows; package explorer; etc (all my views) are blank. I can see line numbers at left; and if I open; e.g. the updater window; I can see the contents without any problems. Since this was my first time using eclipse on this system; I tried 4.3 and all views were displaying contents. Then I started 4.3 with SWT_GTK3=1 and all contents were gone. So this is something to do with GTK3 but I have no idea of what else should I use to debug the issue.</td>\n",
              "      <td>CLOSED</td>\n",
              "      <td>NDUPLICATE</td>\n",
              "      <td>4.4</td>\n",
              "      <td>2013-12-28 8:57</td>\n",
              "      <td>2014-01-04 13:54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85153</th>\n",
              "      <td>424722</td>\n",
              "      <td>P3</td>\n",
              "      <td>Website</td>\n",
              "      <td>NaN</td>\n",
              "      <td>software</td>\n",
              "      <td>NaN</td>\n",
              "      <td>RESOLVED</td>\n",
              "      <td>INVALID</td>\n",
              "      <td>4.3.1</td>\n",
              "      <td>2013-12-29 3:57</td>\n",
              "      <td>2014-01-07 8:47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85154</th>\n",
              "      <td>424764</td>\n",
              "      <td>P3</td>\n",
              "      <td>SWT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Crash (MacOS) - getIvar</td>\n",
              "      <td>Process:         eclipse [42412]; Path:            /Applications/eclipse.app/Eclipse.app/Contents/MacOS/eclipse; Identifier:      org.eclipse.sdk.ide; Version:         4.4.0 (4.4.0.I20131212-1600); Code Type:       X86-64 (Native); Parent Process:  ??? [1]; Responsible:     eclipse [42412]; User ID:         501; ; Date/Time:       2013-12-30 13:35:05.349 -0800; OS Version:      Mac OS X 10.9.1 (13B42); Report Version:  11; Anonymous UUID:  A1947271-CB55-A569-0A50-6F98CD512F75; ; Sleep/Wake U...</td>\n",
              "      <td>CLOSED</td>\n",
              "      <td>NDUPLICATE</td>\n",
              "      <td>4.4</td>\n",
              "      <td>2013-12-30 16:43</td>\n",
              "      <td>2013-12-31 4:35</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85155</th>\n",
              "      <td>424767</td>\n",
              "      <td>P3</td>\n",
              "      <td>IDE</td>\n",
              "      <td>NaN</td>\n",
              "      <td>eclipse</td>\n",
              "      <td>NaN</td>\n",
              "      <td>CLOSED</td>\n",
              "      <td>INVALID</td>\n",
              "      <td>3.2.2</td>\n",
              "      <td>2013-12-30 18:05</td>\n",
              "      <td>2013-12-31 4:21</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>85156 rows × 11 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Issue_id Priority Component  ...  Version      Created_time     Resolved_time\n",
              "0             1       P3      Team  ...        2  2001-10-10 21:34  2012-02-09 15:57\n",
              "1             2       P5      Team  ...        2  2001-10-10 21:34  2002-05-07 10:33\n",
              "2             3       P5      Team  ...        2  2001-10-10 21:34  2010-05-07 10:28\n",
              "3             4       P5      Team  ...        2  2001-10-10 21:34  2002-03-01 16:27\n",
              "4             5       P3      Team  ...        2  2001-10-10 21:34   2008-08-15 8:04\n",
              "...         ...      ...       ...  ...      ...               ...               ...\n",
              "85151    424679       P3       IDE  ...      4.3  2013-12-26 20:12   2013-12-31 5:32\n",
              "85152    424714       P3        UI  ...      4.4   2013-12-28 8:57  2014-01-04 13:54\n",
              "85153    424722       P3   Website  ...    4.3.1   2013-12-29 3:57   2014-01-07 8:47\n",
              "85154    424764       P3       SWT  ...      4.4  2013-12-30 16:43   2013-12-31 4:35\n",
              "85155    424767       P3       IDE  ...    3.2.2  2013-12-30 18:05   2013-12-31 4:21\n",
              "\n",
              "[85156 rows x 11 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "803Wm-1nsJSb",
        "colab_type": "code",
        "outputId": "bd8e2e59-7560-4770-a6cd-749e9f773c2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 756
        }
      },
      "source": [
        "#Removing the unuseful coloumns from the dataset\n",
        "data = data.drop(columns=['Priority', 'Component', 'Status', 'Resolution', 'Version', 'Created_time', 'Resolved_time'], axis=1)\n",
        "data"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Issue_id</th>\n",
              "      <th>Duplicated_issue</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Usability issue with external editors (1GE6IRL)</td>\n",
              "      <td>- Setup a project that contains a *.gif resource; \\t- release project to CVS; \\t- edit the *.gif resource with an external editor; e.g. PaintShop; \\t- save and close external editor; \\t- in Navigator open the icon resource and verify that your changes are there; \\t- release project; \\t\\t-&gt; nothing to release!; \\t- in Navigator open the icon resource and verify that your changes are still there; ; \\tProblem: because I never Refreshed from local; the workspace hasnt changed so Release didnt fi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Opening repository resources doesnt honor type (1GE6I1Q)</td>\n",
              "      <td>Opening repository resource always open the default text editor and doesnt; \\thonor any mapping between resource types and editors.; \\tAs a result it is not possible to view the contents of an image (*.gif file) in a sensible way.; ; NOTES:; EG (5/24/2001 5:13:45 AM); \\tAgreed that this is needed eventually but not critical for June. \\t; \\tThere should also be a Java outliner when editing a java resource from the repository.; \\tMoving to VCM.; ; KM (5/24/01 2:09:21 PM); \\tIn general you cann...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Sync does not indicate deletion (1GIEN83)</td>\n",
              "      <td>KM (10/2/2001 5:55:18 PM); \\tThis PR about the deletion indicator in the sync viewer being too subtle.; \\tThis would be even more true for someone with vision problems.; \\tNE has good suggestion of using the label of the compare text viewer.; \\tWe may also consider a better icon.; \\tOn to the PR....; ; - delete file from project; - synchronize; - there is no indication that the file has been deleted from the workspace; - it looks the same as if the file is in the workspace but is empty; ; NO...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>need better error message if catching up over read-only resource (1GF69TF)</td>\n",
              "      <td>- become synchronized with some project in a repository; - use a different Eclipse to make a change to a file resource within this project and release it to the repository; - in the original Eclipse mark this file resource as being read-only (select it; right-click -&gt; Properties; change; OK); - select the file resource; right-click -&gt; Team -&gt; Synchronize with Stream; - in the subsequent comparison view select the file resource; right-click -&gt; Catchup; - since it has been marked as read-only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>ISharingManager sharing API inconsistent (1GAUL8H)</td>\n",
              "      <td>For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85151</th>\n",
              "      <td>424679</td>\n",
              "      <td>390071.0</td>\n",
              "      <td>Crash on bootstarp</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85152</th>\n",
              "      <td>424714</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[GTK/Linux] Blank Windows with GTK3</td>\n",
              "      <td>I started testing 4.4 and noticed the editor windows; package explorer; etc (all my views) are blank. I can see line numbers at left; and if I open; e.g. the updater window; I can see the contents without any problems. Since this was my first time using eclipse on this system; I tried 4.3 and all views were displaying contents. Then I started 4.3 with SWT_GTK3=1 and all contents were gone. So this is something to do with GTK3 but I have no idea of what else should I use to debug the issue.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85153</th>\n",
              "      <td>424722</td>\n",
              "      <td>NaN</td>\n",
              "      <td>software</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85154</th>\n",
              "      <td>424764</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Crash (MacOS) - getIvar</td>\n",
              "      <td>Process:         eclipse [42412]; Path:            /Applications/eclipse.app/Eclipse.app/Contents/MacOS/eclipse; Identifier:      org.eclipse.sdk.ide; Version:         4.4.0 (4.4.0.I20131212-1600); Code Type:       X86-64 (Native); Parent Process:  ??? [1]; Responsible:     eclipse [42412]; User ID:         501; ; Date/Time:       2013-12-30 13:35:05.349 -0800; OS Version:      Mac OS X 10.9.1 (13B42); Report Version:  11; Anonymous UUID:  A1947271-CB55-A569-0A50-6F98CD512F75; ; Sleep/Wake U...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85155</th>\n",
              "      <td>424767</td>\n",
              "      <td>NaN</td>\n",
              "      <td>eclipse</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>85156 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Issue_id  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          Description\n",
              "0             1  ...  - Setup a project that contains a *.gif resource; \\t- release project to CVS; \\t- edit the *.gif resource with an external editor; e.g. PaintShop; \\t- save and close external editor; \\t- in Navigator open the icon resource and verify that your changes are there; \\t- release project; \\t\\t-> nothing to release!; \\t- in Navigator open the icon resource and verify that your changes are still there; ; \\tProblem: because I never Refreshed from local; the workspace hasnt changed so Release didnt fi...\n",
              "1             2  ...  Opening repository resource always open the default text editor and doesnt; \\thonor any mapping between resource types and editors.; \\tAs a result it is not possible to view the contents of an image (*.gif file) in a sensible way.; ; NOTES:; EG (5/24/2001 5:13:45 AM); \\tAgreed that this is needed eventually but not critical for June. \\t; \\tThere should also be a Java outliner when editing a java resource from the repository.; \\tMoving to VCM.; ; KM (5/24/01 2:09:21 PM); \\tIn general you cann...\n",
              "2             3  ...  KM (10/2/2001 5:55:18 PM); \\tThis PR about the deletion indicator in the sync viewer being too subtle.; \\tThis would be even more true for someone with vision problems.; \\tNE has good suggestion of using the label of the compare text viewer.; \\tWe may also consider a better icon.; \\tOn to the PR....; ; - delete file from project; - synchronize; - there is no indication that the file has been deleted from the workspace; - it looks the same as if the file is in the workspace but is empty; ; NO...\n",
              "3             4  ...  - become synchronized with some project in a repository; - use a different Eclipse to make a change to a file resource within this project and release it to the repository; - in the original Eclipse mark this file resource as being read-only (select it; right-click -> Properties; change; OK); - select the file resource; right-click -> Team -> Synchronize with Stream; - in the subsequent comparison view select the file resource; right-click -> Catchup; - since it has been marked as read-only ...\n",
              "4             5  ...                                        For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:\n",
              "...         ...  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  ...\n",
              "85151    424679  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  NaN\n",
              "85152    424714  ...       I started testing 4.4 and noticed the editor windows; package explorer; etc (all my views) are blank. I can see line numbers at left; and if I open; e.g. the updater window; I can see the contents without any problems. Since this was my first time using eclipse on this system; I tried 4.3 and all views were displaying contents. Then I started 4.3 with SWT_GTK3=1 and all contents were gone. So this is something to do with GTK3 but I have no idea of what else should I use to debug the issue.\n",
              "85153    424722  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  NaN\n",
              "85154    424764  ...  Process:         eclipse [42412]; Path:            /Applications/eclipse.app/Eclipse.app/Contents/MacOS/eclipse; Identifier:      org.eclipse.sdk.ide; Version:         4.4.0 (4.4.0.I20131212-1600); Code Type:       X86-64 (Native); Parent Process:  ??? [1]; Responsible:     eclipse [42412]; User ID:         501; ; Date/Time:       2013-12-30 13:35:05.349 -0800; OS Version:      Mac OS X 10.9.1 (13B42); Report Version:  11; Anonymous UUID:  A1947271-CB55-A569-0A50-6F98CD512F75; ; Sleep/Wake U...\n",
              "85155    424767  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  NaN\n",
              "\n",
              "[85156 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XzYpjWP-sJSl",
        "colab_type": "code",
        "outputId": "71ec96c8-6550-4f52-939e-81136f149bf5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Printing the Coloumn names\n",
        "data.keys()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['Issue_id', 'Duplicated_issue', 'Title', 'Description'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YYPZOGe7sJSw",
        "colab_type": "code",
        "outputId": "37d7f937-9fe1-484d-bc6e-ccaad5ba3e85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Checking the shape of the data\n",
        "data.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(85156, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uGcxxpIswWt2",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## **1. Text Cleaning**\n",
        "\n",
        "  *   Removing Invalid Reports\n",
        "  *   Removing Punctuations\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R_215ZlBsJTC",
        "colab_type": "code",
        "outputId": "a6a14a75-f6ac-40a3-cdab-d2b49833d963",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Number of empty Description reports\n",
        "data['Description'].isnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "129"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zd8ECNy9sJTX",
        "colab_type": "code",
        "outputId": "6a705251-88e4-46ed-9daa-a9aab82b9e18",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Number of empty Title reports\n",
        "data['Title'].isnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hJYYLLR2sJTg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#removing the rows with empty Description\n",
        "data = data.dropna(axis=0, subset=['Description'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ykwl8aH0sJUH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pd.options.display.max_colwidth = 500"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iZJCdlETsJUN",
        "colab_type": "code",
        "outputId": "aa558453-7203-48a5-bfd0-9215b1ad1557",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "#Showing the fixed pattern used in many reports which doesn't have to do anything with the bug itself\n",
        "# Eg: fixed in head, has been marked as read only\n",
        "data.loc[[4]]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Issue_id</th>\n",
              "      <th>Duplicated_issue</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>ISharingManager sharing API inconsistent (1GAUL8H)</td>\n",
              "      <td>For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Issue_id  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Description\n",
              "4         5  ...  For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:\n",
              "\n",
              "[1 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mRdThicxsJUX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing the fixed patterns by replacing them with \" \"\n",
        "data[\"Description\"]= data[\"Description\"].str.replace(\"fixed in HEAD\", \"\", case = False)\n",
        "data[\"Description\"]= data[\"Description\"].str.replace(\"has been marked as readonly\", \" \", case = False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D8Jqn-nrsJUe",
        "colab_type": "code",
        "outputId": "4b49ba2e-c098-4393-a851-0b500e165d64",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "#Checking if it is been removed\n",
        "data.loc[[4]]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Issue_id</th>\n",
              "      <th>Duplicated_issue</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>ISharingManager sharing API inconsistent (1GAUL8H)</td>\n",
              "      <td>For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Issue_id  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Description\n",
              "4         5  ...  For getting/setting the managed state of a resource; the methods are:; ; isManaged(resource) and; manage(resource); ; for getting/setting the ignore state; the methods are:; ; getIgnored(resource); setIgnored(resource); ; These should be made more consistent.  I suggest renaming ignore methods:; ; isIgnored(resource) and; ignore(resource).; ; I think its good practice to not use get and set in method names unless its really; just a field accessor.; ; NOTES:\n",
              "\n",
              "[1 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7HFyT1x3sJU8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pd.options.display.max_colwidth = 50"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-vXj9fxsJTx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text Cleaning round 1 (removing punctutions)\n",
        "import re\n",
        "import string\n",
        "\n",
        "def clean_text_round1(text):\n",
        "    '''Make text lowercase, remove text in square brackets, remove punctuation and remove words containing numbers.'''\n",
        "    text = re.sub('\\w*\\d\\w*', '', text)\n",
        "    text = re.sub('\\w*\\f\\w*', '', text)\n",
        "    text = re.sub('\\(.*?\\)', '', text)\n",
        "    text = re.sub('\\[.*]\\)', '', text)\n",
        "    text = text.lower()\n",
        "    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)\n",
        "    return text\n",
        "\n",
        "round1 = lambda x: clean_text_round1(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4tbSDTdssJT5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['Description'] = data['Description'].apply(clean_text_round1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZBWtQDrsJUo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Apply a second round of cleaning (removing punctuations)\n",
        "def clean_text_round2(text):\n",
        "    '''Get rid of some additional punctuation and non-sensical text that was missed the first time around.'''\n",
        "    text = re.sub('[‘’“”…]', '', text)\n",
        "    text = re.sub('\\n', '', text)\n",
        "    text = re.sub('\\t', '', text)\n",
        "    return text\n",
        "\n",
        "round2 = lambda x: clean_text_round2(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RgcRhy89sJUu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['Title'] = data['Title'].apply(clean_text_round2)\n",
        "data['Description'] = data['Description'].apply(clean_text_round2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "USiMQbfkzamT",
        "colab_type": "text"
      },
      "source": [
        "## **2. Stop Word Removal, Tokenization and Lemmatization**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hBURqTvQsJVQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def lemmatize(text):\n",
        "    return WordNetLemmatizer().lemmatize(text, pos='v')\n",
        "\n",
        "def preprocess(text):\n",
        "    result = []\n",
        "    for token in gensim.utils.simple_preprocess(text):\n",
        "        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 5:\n",
        "            result.append(lemmatize(token))\n",
        "    return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GqEYgHa9sJVU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['Title'] = data['Title'].map(preprocess)\n",
        "data['Description'] = data['Description'].map(preprocess)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n7kChz6UsJVZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data.to_csv(\"drive/My Drive/duplicate_detection/preprocessed_data.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4A0SfiKI_bOB",
        "colab_type": "text"
      },
      "source": [
        "## **Seperation of Master Reports and Duplicate Reports**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mol4vTQs_aam",
        "colab_type": "code",
        "outputId": "e3a4372b-3ad2-4ac1-acf5-70d406f794c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Number of master_reports\n",
        "data['Duplicated_issue'].isnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "70629"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DItsOj4P_ofY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Saving all the duplicate reports into a csv file as a testing set\n",
        "duplicate_reports = data.dropna(axis=0, subset=['Duplicated_issue'])\n",
        "duplicate_reports.reset_index(drop=True)\n",
        "duplicate_reports.to_csv('drive/My Drive/duplicate_detection/duplicate_reports.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sTPKJWYr_sOP",
        "colab_type": "code",
        "outputId": "249bda8c-46d4-4c6a-9efa-56714b6f7157",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        }
      },
      "source": [
        "#Seperating all the master reports into a dataframe\n",
        "master_reports = data[data.isnull().any(axis=1)]\n",
        "master_reports.reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Issue_id</th>\n",
              "      <th>Duplicated_issue</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[usability, external, editors]</td>\n",
              "      <td>[project, contain, resource, release, project,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[open, repository, resources, doesnt]</td>\n",
              "      <td>[open, repository, resource, default, editor, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[indicate, deletion]</td>\n",
              "      <td>[deletion, indicator, viewer, subtle, vision, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[better, message, catch, resource]</td>\n",
              "      <td>[synchronize, project, repository, different, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[isharingmanager, share, inconsistent]</td>\n",
              "      <td>[gettingsetting, manage, resource, methods, is...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70624</th>\n",
              "      <td>424656</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[fail, relate, config]</td>\n",
              "      <td>[feel, relate, need, change, relate, copy, bel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70625</th>\n",
              "      <td>424658</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[eclipse, kepler, ubuntu, jrockit]</td>\n",
              "      <td>[eclipse, kepler, ubuntu, oracle, jrockit, att...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70626</th>\n",
              "      <td>424672</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[display]</td>\n",
              "      <td>[button, dropdown, doesnt, button, result, but...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70627</th>\n",
              "      <td>424714</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[windows]</td>\n",
              "      <td>[start, test, notice, editor, windows, package...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70628</th>\n",
              "      <td>424764</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[getivar]</td>\n",
              "      <td>[process, eclipse, identifier, version, parent...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>70629 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Issue_id  ...                                        Description\n",
              "0             1  ...  [project, contain, resource, release, project,...\n",
              "1             2  ...  [open, repository, resource, default, editor, ...\n",
              "2             3  ...  [deletion, indicator, viewer, subtle, vision, ...\n",
              "3             4  ...  [synchronize, project, repository, different, ...\n",
              "4             5  ...  [gettingsetting, manage, resource, methods, is...\n",
              "...         ...  ...                                                ...\n",
              "70624    424656  ...  [feel, relate, need, change, relate, copy, bel...\n",
              "70625    424658  ...  [eclipse, kepler, ubuntu, oracle, jrockit, att...\n",
              "70626    424672  ...  [button, dropdown, doesnt, button, result, but...\n",
              "70627    424714  ...  [start, test, notice, editor, windows, package...\n",
              "70628    424764  ...  [process, eclipse, identifier, version, parent...\n",
              "\n",
              "[70629 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0N1w_5iN_vVc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing empty master_reports after data cleaning process\n",
        "mr = pd.DataFrame()\n",
        "count = 0\n",
        "for i in range(len(master_reports)):\n",
        "    if(len(master_reports.Description[i])>2):\n",
        "        mr= mr.append(master_reports.loc[[i]])\n",
        "        count=+1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p7kUWR34_xFQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Saving all the master reports into a seperate csv file\n",
        "master_reports.to_csv('drive/My Drive/duplicate_detection/master_reports.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}