text mining in python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"Ethics are built right into the ideals and objectives of the United Nations\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "text2 = text1.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ethics',\n",
       " 'are',\n",
       " 'built',\n",
       " 'right',\n",
       " 'into',\n",
       " 'the',\n",
       " 'ideals',\n",
       " 'and',\n",
       " 'objectives',\n",
       " 'of',\n",
       " 'the',\n",
       " 'United',\n",
       " 'Nations']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ethics',\n",
       " 'built',\n",
       " 'right',\n",
       " 'into',\n",
       " 'ideals',\n",
       " 'objectives',\n",
       " 'United',\n",
       " 'Nations']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in text2 if len(w)>3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ethics', 'United', 'Nations']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Capitalized words\n",
    "[w for w in text2 if w.istitle()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ethics', 'ideals', 'objectives', 'Nations']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# words that end with s \n",
    "[w for w in text2 if w.endswith('s')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "text3 = 'To be or not to be' # findind unique words\n",
    "text4  =  text3.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['To', 'be', 'or', 'not', 'to', 'be']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(text4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'To', 'be', 'not', 'or', 'to'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(text4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set([w.lower() for w in text4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'be', 'not', 'or', 'to'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set([w.lower() for w in text4 ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some words comparison functions..\n",
    "# s.startswith(t)\n",
    "#s.endswith(t)\n",
    "# t in s\n",
    "# s.isupper() ; s.islower(); s.istitle()\n",
    "# s.isalpha( ); s.isdigit(); s.isalnum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# string Operation \n",
    "# s.lower()\n",
    "# s.upper();\n",
    "# s.titlecase();\n",
    "# s.split(t)\n",
    "# s.splitlines()\n",
    "# s.join(t)\n",
    "# s.strip()\n",
    "# s.rstrip()\n",
    "# s.find(t); s.rfind(t)\n",
    "# s.replace(u,v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "text5 = 'ouagadougou'\n",
    "text6= text5.split('ou')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', 'agad', 'g', '']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ouagadougou'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'ou'.join(text6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['o', 'u', 'a', 'g', 'a', 'd', 'o', 'u', 'g', 'o', 'u']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(text5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['o', 'u', 'a', 'g', 'a', 'd', 'o', 'u', 'g', 'o', 'u']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in text5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "text8 = '   A quick brown fox jumped over the lazy dog.  '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '',\n",
       " '',\n",
       " 'A',\n",
       " 'quick',\n",
       " 'brown',\n",
       " 'fox',\n",
       " 'jumped',\n",
       " 'over',\n",
       " 'the',\n",
       " 'lazy',\n",
       " 'dog.',\n",
       " '',\n",
       " '']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text8.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "text9 = text8.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A quick brown fox jumped over the lazy dog.'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog.']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text9.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A quick brown fox jumped over the lazy dog.'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# changing text\n",
    "# Find and replace\n",
    "text9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text9.find('o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text9.rfind('o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A quick brOwn fOx jumped Over the lazy dOg.'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text9.replace('o', 'O')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# handling large texts\n",
    "f=open('C:\\\\Users\\\\ujjwal\\\\Desktop\\\\docker.txt', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Day 13\\n'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.readline()# reading line by line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading the full file \n",
    "f.seek(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "text12= f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Day 13\\n\\nPortainer in docker \\n\\n1. docker run -itd --name web -p 7777:9000 -v /var/run/docker.sock:/var/run/docker.sock portainer/portainer\\n2. docker ps\\n3. docker logs web\\n4. docker logs -f web # for stop this use ctrl+c\\n\\nnow check your ip and enter in browser:7777\\n \\n\\nmysql\\n\\n1. docker run -itd -e MYSQL_ROOT_PASSWORD=redhat -p 1122:3306  --name okdb   mysql\\n2. docker ps\\n3. docker logs okdb\\n4. docker logs -f okdb\\n5. docker exec -it okdb bash\\n\\t show databases;\\n\\t exit;\\n6. docker ps\\n\\n\\ncompose file \\n \\n1. mkdir /tmp/okk\\n2. cd /tmp/okk\\n3. vim docker-compose.yml\\n    \\tversion: \\'3.1\\'\\n\\tsevices:\\n\\t  db:\\n\\t    images: mysql\\n\\t    command: --default-authentication-plugin=mysql_native_password\\n\\t    restart: always\\n\\t    environment:\\n\\t      MYSQL_ROOT_PASSWORD: redhat\\n\\t  \\n\\t  adminer:\\n\\t     image: adminer #web server image like nginx\\n             restart: always\\n\\t     depends_on:\\n\\t       - db\\n              ports: \\n\\t\\t- 8080:8080\\n\\npress esc then write :wp\\n\\n4. docker-compose up -d\\n5. docker compose ps\\n\\nnow copy ip of your and type in browser ip:8080\\n\\n\\n\\nDay 12: ----\\n\\n\\ndockerfile + compose\\n\\n1. mkdir ujjwlapp1\\n2. cd ujjwalapp1 \\n3. git clone https://github.com/mdn/beginner-html-site-styled\\n4. ls\\n5. mv beginner-html-site-styled/ web1\\n6. ls\\n7. vim Dockerfile\\n\\tFROM centos\\n\\tMAINTAINER ujjwalsha768@gmail.com\\n\\tENV x=web1\\n\\tRUN dnf install httpd -y\\n\\t# you can run dnf instead of yum\\n\\tADD web1 /var/www/html/\\n\\tEXPOSE 80\\n\\tENTRYPOINT httpd -DFOREGROUND\\n   :wq\\n\\n8. ls\\n9. vim docker-compose.yml\\n\\tversion: \\'3.8\\'\\n\\tservices:\\n\\t  app1:\\n\\t    image: ujjwal:day12v1  #here image that i want to build \\n\\t    build:  #location of docker file \\n\\t      context: .  #location of docker file\\n\\t      dockerfile: ujjwal.dockerfile    #name of docker file\\n\\t    container_name: ujjwal12c1  #pure optional field\\n\\t    ports: \\n\\t      - \"9991:80\"\\n\\t    network: \\n\\t      - ujjwal11  # i want to use this bridge but this is not present as of now\\n\\t    volumes: \\n\\t      - ujjwalvol12:/mnt/google  #here we are attaching it \\t\\n\\t networks:\\n\\t  ujjwal11:   # this will create a new docker network\\n\\t \\n\\t volumes:\\n\\t  ujjwal11:   #this wil create a new volume\\n\\n\\n\\t:wq\\n\\n10. ls \\n11. docker-compose up -d\\n12. docker-compose ps\\n13. docker-compose images\\n14. now your ip and: 9991 in browser\\n15. docker-compose down\\n16. docker-compose up -d\\nnow change something in your html file its not change so how to slow this type problem\\n\\n17.  docker-compose up --build -d\\nnow again check website its change\\t\\t'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2433"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "text13 = text12.splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "113"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text13)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Day 13'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text13[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file operatios \n",
    "# f= open(filename,  mode)\n",
    "# f.readlie();f.read(); f.read(n)\n",
    "# for line in f: doSomething(line)\n",
    "# f.seek()\n",
    "# f.write(message)\n",
    "# f.close()\n",
    "# f.closed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('C:\\\\Users\\\\ujjwal\\\\Desktop\\\\docker.txt', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "text14 = f.readline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Day 13\\n'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text14 # issues with reading text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Day 13'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how do you remove the last newline character ?\n",
    "text14.rstrip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Processing free-text\n",
    "text15 = '\"Ethics are built right into the ideals and objectives of the United Nations\" \\\n",
    "#UNSG @ NY Society for Ethical Culture bit.ly/2guVelr'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "text16 = text15.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\"Ethics',\n",
       " 'are',\n",
       " 'built',\n",
       " 'right',\n",
       " 'into',\n",
       " 'the',\n",
       " 'ideals',\n",
       " 'and',\n",
       " 'objectives',\n",
       " 'of',\n",
       " 'the',\n",
       " 'United',\n",
       " 'Nations\"',\n",
       " '#UNSG',\n",
       " '@',\n",
       " 'NY',\n",
       " 'Society',\n",
       " 'for',\n",
       " 'Ethical',\n",
       " 'Culture',\n",
       " 'bit.ly/2guVelr']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['#UNSG']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in text16 if w.startswith('#')] # fiding hastags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in text16 if w.startswith('@')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "text17 = '@UN @UN_Women \"Ethics are built right into the ideals and objectives of the United Nations\" \\#UNSG @ NY Society for Ethical Culture bit.ly/2guVelr'\n",
    "text18 = text17.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@UN',\n",
       " '@UN_Women',\n",
       " '\"Ethics',\n",
       " 'are',\n",
       " 'built',\n",
       " 'right',\n",
       " 'into',\n",
       " 'the',\n",
       " 'ideals',\n",
       " 'and',\n",
       " 'objectives',\n",
       " 'of',\n",
       " 'the',\n",
       " 'United',\n",
       " 'Nations\"',\n",
       " '\\\\#UNSG',\n",
       " '@',\n",
       " 'NY',\n",
       " 'Society',\n",
       " 'for',\n",
       " 'Ethical',\n",
       " 'Culture',\n",
       " 'bit.ly/2guVelr']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We can use regular expressions to help us with more complex parsing.\n",
    "\n",
    "#For example '@[A-Za-z0-9_]+' will return all words that:\n",
    "\n",
    "#start with '@' and are followed by at least one:\n",
    "#capital letter ('A-Z')\n",
    "#lowercase letter ('a-z')\n",
    "#number ('0-9')\n",
    "#or underscore ('_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import re  # import re - a module that provides support for regular expressions\n",
    "#[w for w in text18 if re.search('@[A-Za-z0-9_]+',w)]\n",
    "\n",
    "\n",
    "\n",
    "########## \n",
    "#######  Meta-chacters: Character matches\n",
    "\n",
    "#  . \"wildcaard, matches a single character\" \n",
    "#  ^ \"start of a string\"\n",
    "# $ \"end of a string\"\n",
    "# [] \"matches one of the set of character within []\"\n",
    "# [a-z] \"matches one of the range of chacarters a,b,c,....z\"\n",
    "# [^abc] \"matches a character that is not a,b, or c \"\n",
    "# a|b \"matches either a or  b, where a and b are strings\"\n",
    "# () \"Scoping for opertors\"\n",
    "# \\ \"Escape character for special character (\\t,\\n,\\b)\"\n",
    "\n",
    "########## \n",
    "#######  Meta-chacters: Character symbols\n",
    "\n",
    "#  \\b \"matches word boundary\"\n",
    "#  \\d \"any digit, equivalent to [0-9]\"\n",
    "#   \\D \"any non-digit, equivalent to [^0-9]\"\n",
    "#  \\s \"any white space, equivalent to [ \\t \\n \\r \\f \\v]\"\n",
    "#  \\S \"any non-white space, equivalent to[^ \\t\\n\\r\\f\\v]\"\n",
    "# \\w \"Alphaumeric character , equivalent to[a-zA-Z0-9_]\"\n",
    "# \\W \" non-alphanumeric, equivalent to[^a-zA-Z0-9_]\"\n",
    "\n",
    "\n",
    "########## \n",
    "#######  Meta-chacters: Repetitios\n",
    "\n",
    "# * \"matches zero or more occurreces\"\n",
    "# + \" MATCHES one or more occurrences\"\n",
    "# ? \"Matches zero or more occurrences\"\n",
    "# {n} \"exactly n repetitions\"\n",
    "# {n,} \"at least n repetitions\"\n",
    "# {,n} \" at most n repetitions\"\n",
    "# {m,n} \"at least m and at most n repetitions\"\n",
    "\n",
    "#[w for w in text18 if re.search('@\\w+' , w)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['o', 'u', 'u', 'i', 'u', 'e', 'a']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#finding special charcter \n",
    "import re\n",
    "text20 = 'ousnduiuela'\n",
    "re.findall(r'[aeiou]',text20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['s', 'n', 'd', 'l']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'[^aeiou]',text20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#regular expression for  dates\n",
    "dateStr = '23-10-2002\\n23/10/2002\\n23/10/02\\n10/23/2002\\n23 Oct 2002\\n23 October 2002\\nOct 23,2002\\nOctober 23,2002\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23-10-2002', '23/10/2002', '10/23/2002']"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\d{2}[/-]\\d{2}[/-]\\d{4}',dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23-10-2002', '23/10/2002', '23/10/02', '10/23/2002']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\d{2}[/-]\\d{2}[/-]\\d{2,4}',dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23-10-2002', '23/10/2002', '23/10/02', '10/23/2002']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\d{1,2}[/-]\\d{1,2}[/-]\\d{2,4}',dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Oct']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Regex for Dates(Contd.)\n",
    "re.findall(r'\\d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \\d{4}',dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23 Oct 2002']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \\d{4}',dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23 Oct 2002', '23 October 2002']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \\d{4}',dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23 Oct 2002', '23 October 2002', 'Oct 23,2002', 'October 23,2002']"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'(?:\\d{2} )?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* (?:\\d{2},)?\\d{4}', dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['23 Oct 2002', '23 October 2002', 'Oct 23,2002', 'October 23,2002']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'(?:\\d{1,2} )?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* (?:\\d{1,2},)?\\d{4}', dateStr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Working with Text Data in pandas\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Monday: The doctor's appointment is at 2:45pm.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tuesday: The dentist's appointment is at 11:30...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Wednesday: At 7:00pm, there is a basketball game!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thursday: Be back home by 11:15 pm at the latest.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Friday: Take the train at 08:10 am, arrive at ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0     Monday: The doctor's appointment is at 2:45pm.\n",
       "1  Tuesday: The dentist's appointment is at 11:30...\n",
       "2  Wednesday: At 7:00pm, there is a basketball game!\n",
       "3  Thursday: Be back home by 11:15 pm at the latest.\n",
       "4  Friday: Take the train at 08:10 am, arrive at ..."
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_sentences = [\"Monday: The doctor's appointment is at 2:45pm.\", \n",
    "                  \"Tuesday: The dentist's appointment is at 11:30 am.\",\n",
    "                  \"Wednesday: At 7:00pm, there is a basketball game!\",\n",
    "                  \"Thursday: Be back home by 11:15 pm at the latest.\",\n",
    "                  \"Friday: Take the train at 08:10 am, arrive at 09:00am.\"]\n",
    "\n",
    "df = pd.DataFrame(time_sentences, columns=['text'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    46\n",
       "1    50\n",
       "2    49\n",
       "3    49\n",
       "4    54\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the number of characters for each string in df['text']\n",
    "df['text'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     7\n",
       "1     8\n",
       "2     8\n",
       "3    10\n",
       "4    10\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the number of tokens for each string in df['text']\n",
    "df['text'].str.split().str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     True\n",
       "1     True\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "Name: text, dtype: bool"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find which entries contain the word 'appointment'\n",
    "df['text'].str.contains('appointment')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3\n",
       "1    4\n",
       "2    3\n",
       "3    4\n",
       "4    8\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find how many times a digit occurs in each string\n",
    "df['text'].str.count(r'\\d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                   [2, 4, 5]\n",
       "1                [1, 1, 3, 0]\n",
       "2                   [7, 0, 0]\n",
       "3                [1, 1, 1, 5]\n",
       "4    [0, 8, 1, 0, 0, 9, 0, 0]\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find all occurances of the digits\n",
    "df['text'].str.findall(r'\\d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0               [(2, 45)]\n",
       "1              [(11, 30)]\n",
       "2               [(7, 00)]\n",
       "3              [(11, 15)]\n",
       "4    [(08, 10), (09, 00)]\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# group and find the hours and minutes\n",
    "df['text'].str.findall(r'(\\d?\\d):(\\d\\d)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          ???: The doctor's appointment is at 2:45pm.\n",
       "1       ???: The dentist's appointment is at 11:30 am.\n",
       "2          ???: At 7:00pm, there is a basketball game!\n",
       "3         ???: Be back home by 11:15 pm at the latest.\n",
       "4    ???: Take the train at 08:10 am, arrive at 09:...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace weekdays with '???'\n",
    "df['text'].str.replace(r'\\w+day\\b', '???')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          Mon: The doctor's appointment is at 2:45pm.\n",
       "1       Tue: The dentist's appointment is at 11:30 am.\n",
       "2          Wed: At 7:00pm, there is a basketball game!\n",
       "3         Thu: Be back home by 11:15 pm at the latest.\n",
       "4    Fri: Take the train at 08:10 am, arrive at 09:...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace weekdays with 3 letter abbrevations\n",
    "df['text'].str.replace(r'(\\w+day\\b)', lambda x: x.groups()[0][:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1\n",
       "0   2  45\n",
       "1  11  30\n",
       "2   7  00\n",
       "3  11  15\n",
       "4  08  10"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create new columns from first match of extracted groups\n",
    "df['text'].str.extract(r'(\\d?\\d):(\\d\\d)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>match</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>2:45pm</td>\n",
       "      <td>2</td>\n",
       "      <td>45</td>\n",
       "      <td>pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <td>11:30 am</td>\n",
       "      <td>11</td>\n",
       "      <td>30</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <td>7:00pm</td>\n",
       "      <td>7</td>\n",
       "      <td>00</td>\n",
       "      <td>pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>0</th>\n",
       "      <td>11:15 pm</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
       "      <th>0</th>\n",
       "      <td>08:10 am</td>\n",
       "      <td>08</td>\n",
       "      <td>10</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>09:00am</td>\n",
       "      <td>09</td>\n",
       "      <td>00</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                0   1   2   3\n",
       "  match                      \n",
       "0 0        2:45pm   2  45  pm\n",
       "1 0      11:30 am  11  30  am\n",
       "2 0        7:00pm   7  00  pm\n",
       "3 0      11:15 pm  11  15  pm\n",
       "4 0      08:10 am  08  10  am\n",
       "  1       09:00am  09  00  am"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the entire time, the hours, the minutes, and the period\n",
    "df['text'].str.extractall(r'((\\d?\\d):(\\d\\d) ?([ap]m))')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>hour</th>\n",
       "      <th>minute</th>\n",
       "      <th>period</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>match</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>2:45pm</td>\n",
       "      <td>2</td>\n",
       "      <td>45</td>\n",
       "      <td>pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <td>11:30 am</td>\n",
       "      <td>11</td>\n",
       "      <td>30</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <td>7:00pm</td>\n",
       "      <td>7</td>\n",
       "      <td>00</td>\n",
       "      <td>pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>0</th>\n",
       "      <td>11:15 pm</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
       "      <th>0</th>\n",
       "      <td>08:10 am</td>\n",
       "      <td>08</td>\n",
       "      <td>10</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>09:00am</td>\n",
       "      <td>09</td>\n",
       "      <td>00</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             time hour minute period\n",
       "  match                             \n",
       "0 0        2:45pm    2     45     pm\n",
       "1 0      11:30 am   11     30     am\n",
       "2 0        7:00pm    7     00     pm\n",
       "3 0      11:15 pm   11     15     pm\n",
       "4 0      08:10 am   08     10     am\n",
       "  1       09:00am   09     00     am"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract the entire time, the hours, the minutes, and the period with group names\n",
    "df['text'].str.extractall(r'(?P<time>(?P<hour>\\d?\\d):(?P<minute>\\d\\d) ?(?P<period>[ap]m))')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}