08_word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Inspired by https://www.tensorflow.org/versions/r0.7/tutorials/word2vec/index.html\n",
    "import collections\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Word count', [('cats', 10), ('dogs', 6), ('and', 5), ('are', 4), ('love', 3)])\n",
      "('Sample data', [8, 33, 24, 20, 17, 12, 8, 25, 30, 26], ['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog', 'I'])\n",
      "('Context pairs', [[[8, 24], 33], [[33, 20], 24], [[24, 17], 20], [[20, 12], 17], [[17, 8], 12], [[12, 25], 8], [[8, 30], 25], [[25, 26], 30], [[30, 4], 26], [[26, 0], 4]])\n",
      "('skip-gram pairs', [[33, 8], [33, 24], [24, 33], [24, 20], [20, 24]])\n",
      "('Batches (x, y)', ([27, 15, 22], [[0], [28], [18]]))\n"
     ]
    }
   ],
   "source": [
    "# Configuration\n",
    "batch_size = 20\n",
    "# Dimension of the embedding vector. Two too small to get\n",
    "# any meaningful embeddings, but let's make it 2 for simple visualization\n",
    "embedding_size = 2\n",
    "num_sampled = 15    # Number of negative examples to sample.\n",
    "\n",
    "# Sample sentences\n",
    "sentences = [\"the quick brown fox jumped over the lazy dog\",\n",
    "            \"I love cats and dogs\",\n",
    "            \"we all love cats and dogs\",\n",
    "            \"cats and dogs are great\",\n",
    "            \"sung likes cats\",\n",
    "            \"she loves dogs\",\n",
    "            \"cats can be very independent\",\n",
    "            \"cats are great companions when they want to be\",\n",
    "            \"cats are playful\",\n",
    "            \"cats are natural hunters\",\n",
    "            \"It's raining cats and dogs\",\n",
    "            \"dogs and cats love sung\"]\n",
    "\n",
    "# sentences to words and count\n",
    "words = \" \".join(sentences).split()\n",
    "count = collections.Counter(words).most_common()\n",
    "print (\"Word count\", count[:5])\n",
    "\n",
    "# Build dictionaries\n",
    "rdic = [i[0] for i in count] #reverse dic, idx -> word\n",
    "dic = {w: i for i, w in enumerate(rdic)} #dic, word -> id\n",
    "voc_size = len(dic)\n",
    "\n",
    "# Make indexed word data\n",
    "data = [dic[word] for word in words]\n",
    "print('Sample data', data[:10], [rdic[t] for t in data[:10]])\n",
    "\n",
    "# Let's make a training data for window size 1 for simplicity\n",
    "# ([the, brown], quick), ([quick, fox], brown), ([brown, jumped], fox), ...\n",
    "cbow_pairs = [];\n",
    "for i in range(1, len(data)-1) :\n",
    "    cbow_pairs.append([[data[i-1], data[i+1]], data[i]]);\n",
    "print('Context pairs', cbow_pairs[:10])\n",
    "\n",
    "# Let's make skip-gram pairs\n",
    "# (quick, the), (quick, brown), (brown, quick), (brown, fox), ...\n",
    "skip_gram_pairs = [];\n",
    "for c in cbow_pairs:\n",
    "    skip_gram_pairs.append([c[1], c[0][0]])\n",
    "    skip_gram_pairs.append([c[1], c[0][1]])\n",
    "print('skip-gram pairs', skip_gram_pairs[:5])\n",
    "\n",
    "def generate_batch(size):\n",
    "    assert size < len(skip_gram_pairs)\n",
    "    x_data=[]\n",
    "    y_data = []\n",
    "    r = np.random.choice(range(len(skip_gram_pairs)), size, replace=False)\n",
    "    for i in r:\n",
    "        x_data.append(skip_gram_pairs[i][0])  # n dim\n",
    "        y_data.append([skip_gram_pairs[i][1]])  # n, 1 dim\n",
    "    return x_data, y_data\n",
    "\n",
    "# generate_batch test\n",
    "print ('Batches (x, y)', generate_batch(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Input data\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "# need to shape [batch_size, 1] for nn.nce_loss\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "# Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "with tf.device('/cpu:0'):\n",
    "    # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs) # lookup table\n",
    "\n",
    "# Construct the variables for the NCE loss\n",
    "nce_weights = tf.Variable(\n",
    "    tf.random_uniform([voc_size, embedding_size],-1.0, 1.0))\n",
    "nce_biases = tf.Variable(tf.zeros([voc_size]))\n",
    "\n",
    "# Compute the average NCE loss for the batch.\n",
    "# This does the magic:\n",
    "#   tf.nn.nce_loss(weights, biases, inputs, labels, num_sampled, num_classes ...)\n",
    "# It automatically draws negative samples when we evaluate the loss.\n",
    "loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, train_labels, embed, num_sampled, voc_size))\n",
    "# Use the adam optimizer\n",
    "train_op = tf.train.AdamOptimizer(1e-1).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Loss at ', 0, 16.654182)\n",
      "('Loss at ', 10, 14.677063)\n",
      "('Loss at ', 20, 9.1576614)\n",
      "('Loss at ', 30, 3.9546738)\n",
      "('Loss at ', 40, 3.8289108)\n",
      "('Loss at ', 50, 3.3630223)\n",
      "('Loss at ', 60, 3.6222715)\n",
      "('Loss at ', 70, 3.1979971)\n",
      "('Loss at ', 80, 3.5327618)\n",
      "('Loss at ', 90, 3.4316573)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgkAAAFkCAYAAACq4KjhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xlc1XXe///HmyNuKEpi2eKCLC5lKkxO5l4khpltMxOg\nebX4nTYX1Mb6XjPtc+VVqWVzVVZmOtT55a3flKUGDZl0uaAGiVk4BzClaUpLDEdLDXh//zhIgB9N\nkHMOy/N+u52b53y29+vz9gPnyWc11lpEREREagsKdAEiIiLSOCkkiIiIiCOFBBEREXGkkCAiIiKO\nFBJERETEkUKCiIiIOFJIEBEREUcKCSIiIuJIIUFEREQcKSSIiIiII5+GBGPMg8aYilqvz33ZpoiI\niDSMVn5oYwdwBWAqP5f5oU0RERE5Q/4ICWXW2m/90I6IiIg0IH+ckxBtjPnKGFNkjEkzxnT3Q5si\nIiJyhowvHxVtjEkAOgD/AM4FHgLOAy6y1h52mL4LkADsBo74rDAREZHmpy3QC8iw1u5viAX6NCSc\n0JgxnYA9QKq1dqnD+GTgNb8VJCIi0vykWGtfb4gF+eOchCrW2lJjjAeIOskkuwHS0tLo16+f3+pq\nDlJTU1m4cGGgy2hS1Gf1o36rO/VZ/ajf6iY/P59JkyZB5XdpQ/BrSDDGdAAigeUnmeQIQL9+/YiN\njfVbXc1Bp06d1Gd1pD6rH/Vb3anP6kf9Vm8Ndrje1/dJeNIYM9IY09MYcxnwFt5LIN2+bFdERETO\nnK/3JFwAvA50Ab4F1gOXNtQJFSIiIuI7Pg0J1tokXy5fREREfEfPbmgmkpKUx+pKfVY/6re6U5/V\nj/ot8Px6CeQvMcbEAjk5OTk6WUVERKQOcnNziYuLA4iz1uY2xDK1J0FEREQcKSSIiIiII4UEERER\ncaSQICIiIo4UEkRERMSRQoKIiIg4UkgQERERRwoJIiIi4kghQURERBwpJIiIiIgjhQQRERFxpJAg\nIiIijhQSRERExJFCgoiIiDhSSBARERFHCgkiIiLiSCFBREREHCkkiIiIiCOFBBEREXGkkCAiIiKO\nFBJERETEkUKCiIiIOFJIEBEREUcKCSIiIuJIIUFEREQcKSSIiIiII4UEERERcaSQICIiIo4UEkRE\nRMSRQoKIiIg4UkgQERERRwoJIiIi4kghQURERBwpJIiIiIgjhQQRERFxpJAgIiIijhQSRERExJFC\ngoiIiDhSSBARERFHfgsJxpj7jTEVxpgF/mpTROR0LVu2jLCwsECXIdKo+CUkGGMuAaYCef5oT0Sk\nPowxgS5BpFHxeUgwxnQA0oDbge993Z6IiIg0DH/sSfgf4F1r7Vo/tCUiLUBGRgYjRowgLCyM8PBw\nJkyYwK5duwDYs2cPQUFBvPXWW1x++eWEhIQwaNAgsrOzayzj1VdfpWfPnnTo0IEbbriB/fv3B2JV\nRBo1n4YEY8xNwCDgfl+2IyIty+HDh5k9ezY5OTmsXbsWl8vFddddV2OaP/7xj/zhD38gLy+PmJgY\nkpOTqaioAGDz5s3cfvvtTJ8+nW3btjFmzBgee+yxQKyKSKNmrLW+WbAxFwAfA1daaz+tHPYh8Im1\ndtZJ5okFckaOHEmnTp1qjEtKSiIpKckntYpI0/btt99yzjnnsGPHDkJCQoiIiOCVV17hP/7jPwDI\nz8/noosuIj8/n5iYGFJSUjh48CDvvvtu1TKSkpLIyMigpKQkQGshcvrcbjdut7vGsNLSUj766COA\nOGttbkO006ohFnIScUBXIMf8fDaQCxhpjLkHaGNPklAWLlxIbGysD0sTkaassLCQBx54gM2bN/Pd\nd99RUVGBMYbi4mL69esHwIABA6qmP/fcc7HWsm/fPmJiYsjPz+f666+vscyhQ4eSkZHh1/UQqS+n\nP5xzc3OJi4tr0HZ8GRIygQG1hr0K5APzThYQRER+ydVXX01ERAQvv/wy5513HuXl5Vx00UUcO3as\naprg4OCq98f/Tjl+uMFaqysZRE6Dz0KCtfYw8Hn1YcaYw8B+a22+r9oVkeatpKQEj8fDkiVLGDZs\nGADr16+v0zL69+9/womMmzZtarAaRZoLX+5JcKK9ByJyRsLCwujSpQsvvvgi3bp1Y8+ePdx///11\n2jMwffp0hg8fzvz585k4cSLp6ek61CDiwK+3ZbbWXn6ykxZFRE6HMYY33niDnJwcBgwYwOzZs3nq\nqaeqxlX/t/Z8x/3617/mpZdeYtGiRQwaNIjMzEz+9Kc/+WcFRJoQn13dUB/Hr27IycnRiYsiIiJ1\nUO3ExSZxdYOISKPk8XgoKioiKiqK6OjoQJcj0mjpKZAi0mKUlJQwbtx4+vTpQ2JiIjExMYwbN54D\nBw4EujSRRkkhQURajOTkyWRmZuN9nEwxkEZmZjZJSZMCXJlI46TDDSLSIng8HjIy1uANCCmVQ1Mo\nL7dkZEymoKBAhx5EatGeBBFpEYqKiirfjaw1ZhTgvYujiNSkkCAiLUJkZGTlu49qjckCICoqyq/1\niDQFCgki0iLExMSQkJCIyzUd7yGHL4E0XK4ZJCQk6lCDiAOFBBFpMdzuNOLjLwUmAz2AycTHX4rb\nnRbgykQaJ524KCItRlhYGOnpqykoKKCwsFD3SRD5BQoJItLiREdHKxyInAYdbhARERFHCgkiIiLi\nSCFBREREHCkkiIiIiCOFBBEREXGkkCAiIiKOFBJERETEkUKCiIiIOFJIEBEREUcKCSIiIuJIIUFE\nREQcKSSIiIiII4UEERERcaSQICIiIo4UEkRERMSRQoKIiIg4UkgQERERRwoJIiIi4kghQURERBwp\nJIiIiIgjhQQRERFxpJAgIiIijhQSRERExJFCgoiIiDhSSBARERFHCgkiIiLiSCFBREREHCkkiIiI\niCOFBBEREXHk05BgjLnDGJNnjCmtfG00xozzZZsiIiLSMHy9J+FLYC4QV/laC6w0xvTzcbsiIiJy\nhlr5cuHW2tW1Bv3RGHMncCmQ78u2RURE5Mz4NCRUZ4wJAn4LtAc2+atdERERqR+fn7hojLnIGPNv\n4CjwHHCdtXanr9ttasaMGcOsWbMCXYaIiEgVf1zdsBMYCPwaeB5Ybozp64d2RURE5Az4/HCDtbYM\n2FX5MdcYMwSYAdx5snlSU1Pp1KlTjWFJSUkkJSX5rE4REZGmwu1243a7awwrLS1t8Hb8dk5CNUFA\nm1NNsHDhQmJjY/1Ujv/98MMP3HHHHbz11luEhoYye/bsGuO///57pk+fzqpVqzh69CijRo1i0aJF\nREVFVU3z0ksv8eijj1JSUkJCQgLDhw/nkUce4cCBAwBs376dmTNn8vHHH2OMISYmhsWLFzfrfhUR\naSmc/nDOzc0lLi6uQdvx9X0S/myMGW6M6Vl5bsLjwCggzZftNnZz5szhf//3f3n33Xd5//33Wbdu\nHTk5OVXjp0yZQm5uLqtWrSI7OxtrLYmJiZSXlwOwYcMG7rzzTlJTU9m2bRtXXnklf/7znzHGVC0j\nJSWF7t27k5OTQ25uLvfddx/BwcF+X1cREWm6fL0n4RxgOXAuUApsB8Zaa9f6uN1G6/Dhw7zyyiu8\n/vrrjB49GoBly5ZxwQUXAFBYWMi7777Lpk2b+PWvfw3Aa6+9Rvfu3Xn77be54YYb+Mtf/kJiYiKp\nqakAREVFsWHDBlav/vmK0+LiYv7whz8QHR0NQGRkpB/XUkREmgOf7kmw1t5ure1trW1nre1mrW3R\nAQGgqKiIn376iSFDhlQNCwsLo0+fPgDk5+cTHBxcY/xZZ51Fnz59yM/33lriH//4R43xwAmfZ82a\nxW233caVV17Jf//3f7Nr1y5ERETqQs9u8DNrLUCNQwNO452GH5+n+vuTzffggw/y+eefc/XVV7N2\n7VouvPBCVq5ceabli4hIC6KQ4GdRUVG0atWK7OzsqmEHDhzA4/EA0L9/f3766Sc2b95cNX7//v14\nPB769+8PQN++fdmyZUuN5W7dutWxrRkzZpCRkcF1113H0qVLfbFKIiLSTCkk+FlISAi33XYb9957\nLx9++CE7duzglltuweVyAd4v9okTJzJ16lQ2bNhAXl4ekyZNonv37lxzzTUATJs2jTVr1rBw4UIK\nCwtZvHgx6enpVXsXjhw5wrRp08jKyqK4uJgNGzawdevWqpAhIiJyOhQSAuDJJ59kxIgRXHPNNYwd\nO5YRI0bUuGxl6dKlxMXFMWHCBIYNG0ZQUBCrV6+uChKXXXYZL7zwAgsXLmTQoEG8//77pKam0rZt\nWwBcLhf79+9nypQp9OnTh5tuuonx48fz0EMPBWJ1RUSkiTInOwYeCMaYWCAnJydH1/PX0dSpU/F4\nPGRlZQW6FBERCYBq90mIs9bmNsQyA3EzJWkA8+fP58orryQkJIQ1a9bw17/+leeffz7QZYmISDOi\nkNBEbdmyhSeffJJ///vf9O7dm2effZZbbrkFAI/HQ1FREVFRUVX3SRAREakrhYQm6o033jhhWElJ\nCcnJk8nIWFM1LCEhEbc7jbCwMH+WJyIizYBOXGxGkpMnk5mZjfeu18VAGpmZ2SQlTQpwZSIi0hRp\nT0Iz4fF4KvcgpAEplUNTKC+3ZGRMpqCgQIceRESkTrQnoZkoKiqqfDey1phRgPeZECIiInWhkNBM\n/PwAp49qjfFeEln9MdMiIiKnQyGhmYiJiSEhIRGXazreQw5fAmm4XDNISEjUoQYREakzhYRmxO1O\nIz7+UmAy0AOYTHz8pbjdaQGuTEREmiKduNiMhIWFkZ6+moKCAgoLC3WfBBEROSMKCc1QdHS0woGI\niJwxHW4QERERRwoJIiIi4kghQURERBwpJIiIiIgjhQQRERFxpJAgIiIijhQSRERExJFCgoiIiDhS\nSBARERFHCgkiIiLiSCFBREREHCkkiIiIiCOFhAb28MMPM3jw4ECXISIicsYUEnzAGHPa0wYFBfHO\nO+/4sBoREZH6UUhwYK3liSeeIDo6mrZt29KrVy8ef/xxAO677z5iYmIICQkhMjKSBx54gPLycgCW\nLVvGww8/TF5eHkFBQbhcLpYvXw7AQw89RM+ePWnbti0XXHABM2fODNj6iYiInA6FBODQoUOkpKTQ\noUMHzj//fOLj4/nP//xPoqKiyM/P59ixY2zZsoUpU6bwzDPP0KdPH/Lz83nggQd48skn6dixI+Hh\n4bz55pvcfvvtXHjhhezdu5f33nuP5cuXExoaysMPP0znzp15++23efvttxkwYAAREREYY7j22msJ\nCgqid+/ege4KERGRKgoJQGpqKps2bWLVqlWsXLmSdevWERwcTL9+/YiIiKBNmzasW7eOQYMG8fnn\nn7No0SLOO+88nnjiCQYOHEhkZCQbNmygc+fOvPXWW7hcLrp27UpwcDC33nord999NxEREQwZMoRb\nbrmFfv36cdttt7F161astSxbtoxvvvmGrVu3BrorREREqrT4kHDo0CGWL1/O/PnzGT16NNZarLUn\nnFdwxRVXkJqaypYtW5g8eTLh4eHk5+eTl5fHt99+S58+fViyZAmlpaX8+9//BmDMmDEkJydz9913\nU15eTnp6Ovv372fevHmUl5cTHh4OQKdOnTj77LPp0qWL39dfRETkZFp8SNi1axdlZWVccsklALRr\n1w5jzAm7/uPi4sjOzmbSpElcffXVTJw4EZfLRUVFBXv37qVjx4506dKFsrIyjh07BsC+ffuYOnUq\nl19+OaWlpezfv5+ysjKeffZZRo0aVXUug4iISGPU4kOCtRb4+YqE4ycrHjx4sMZ0ISEhbNy4kV69\nenHfffcREhLCr371K6699lpCQ0PJy8sjLy+P2bNnExoaCsDNN9/M9u3befbZZ9m0aRM7duwgPDyc\ne+65h40bN/Lpp5/6d2VFRETqoMWHhMjISFq1asWWLVsAaNOmDTNmzKC4uJjPP/+cXbt2cfToUbKz\ns4mOjqa4uJg33niD7t278+mnn5KZmYnL5aJ379707t2b2NhYiouLycvLY8OGDdx111188803bNq0\niV27dvHdd9+xdetW2rdvT8+ePQkODtYeBRERaZRaBbqAQOvQoQNTpkxhzpw5hIWF0bVrVwoKCmjb\nti2bNm2if//+lJeXc+jQISZMmEBqairTpk3jyJEjBAUF0aVLF7755ht2797N7t272bBhA6NGjWLM\nmDEcOnSI+fPn8/vf/57nn3+enTt3Yq2loKCAVatWERYWRq9evfjggw+47LLLaNOmDZ07dw50l4iI\niADakwDAwoULueyyy5gwYQJjx45l+PDhDBw4kHvuuYcjR47QvXt34uPjAZg3bx779u3j4MGDFBYW\nMmzYMNq1a0f//v2ZOnUqZWVluN1uSkpK2LZtG+3atePee++lrKyMFStWEBERwcyZMxk9ejQA8+fP\n5+9//zs9evQgNjY2gL0gIiJSkzl+TL4xMMbEAjk5OTkB/cL84YcfOP/881mwYAG33HJLwOoQERE5\nXbm5ucTFxQHEWWtzG2KZLf5wA8C2bdvYuXMnQ4YM4fvvv+eRRx7BGMPEiRPPeNkej4eioiKioqKI\njo4+6TAREZHGxqchwRhzP3Ad0Bf4EdgIzLXWenzZbn089dRTeDweWrduTVxcHOvXr+ess86q9/JK\nSkpITp5MRsaaqmFjxsRjjGHt2r9XDUtISMTtTiMsLOyM6hcREWlovt6TMAJ4Fvi4sq3HgfeNMf2s\ntT/6uO3TNmjQID7++OMGXWZy8mQyM7OBNGAk8BEffngbxrSrMSwzczpJSZNIT1/doO2LiIicKZ+G\nBGttYvXPxpj/APYBccB6X7YdSB6Pp3IPQhqQUjn0EuAo1i6pNiyF8nJLRsZkCgoKdOhBREQaFX9f\n3dAZsECJn9v1q6Kiosp3I6sPdRgGMAqAwsJCH1clIiJSN34LCcZ7S8OngfXW2s/91W4gREZGVr77\nqPpQh2EAWQBERUX5uCoREZG68efVDc8B/YFhvzRhamoqnTp1qjEsKSmJpKQkH5XWsGJiYkhISCQz\nczrl5Rbv3oItQBuMuafyVtCjgCxcrhnExyfqUIOIiJw2t9uN2+2uMay0tLTB2/HLfRKMMX8BJgAj\nrLXFp5iuUdwnoSEcOHCApKRJNa5uuPzyKwF0dYOIiDS4JnmfhMqAMBEYdaqA0NyEhYWRnr6agoIC\nCgsLa9wTwWmYiIhIY+Pr+yQ8ByQB1wCHjTHnVI4qtdYe8WXbjUV0dPQJQcBpmIiISGPj6xMX7wBC\ngXXAv6q9fuvjdkVEROQM+fo+CXqAlIiISBOlL3ERERFxpJAgIiIijhQSRERExJFCgoiIiDhSSBAR\nERFHCgkiIiLiSCFBREREHCkkiIiIiCOFBBEREXGkkCAiIiKOFBJERETEkUKCiIiIOFJIEBEREUcK\nCSIiIuJIIUFEREQcKSSIiIiII4UEERERcaSQICIiIo4UEkRERMSRQoI0CmVlZYEuQUREalFIEJ/I\nyMhgxIgRhIWFER4ezoQJE9i1axcAe/bsISgoiBUrVjB69Gjat2/P66+/DsD69esZOXIk7du3p2fP\nnsyYMYMffvghkKsiItJiKSSITxw+fJjZs2eTk5PD2rVrcblcXHfddTWmuf/++5k5cyb5+fkkJCSw\na9currrqKn7zm9+wY8cO3njjDTZs2MC0adMCtBYiIi2bsdYGuoYqxphYICcnJ4fY2NhAlyMN6Ntv\nv+Wcc85hx44dhISEEBERwaJFi7jnnnuqppk6dSqtWrXi+eefrxq2fv16Ro8ezQ8//EDr1q0DUbqI\nSJOQm5tLXFwcQJy1Nrchlqk9CeIThYWFJCcnExkZSadOnejduzfGGIqLi6umqdyYq+Tl5fHqq6/S\nsWPHqte4ceMA+OKLL/xav4iIQKtAFyDN09VXX01ERAQvv/wy5513HuXl5Vx00UUcO3asapqQkJAa\n8xw6dIjf//73zJgxg9p7uHr06OGXukVE5GcKCdLgSkpK8Hg8LFmyhGHDhgHewwa/JDY2ls8++4yI\niAhflygiIqdBhxukwYWFhdGlSxdefPFFioqKWLt2LbNnz8YYc8r55s6dy6ZNm5g2bRp5eXkUFhay\ncuVKnbgoIhIgCgnS4IwxvPHGG+Tk5DBgwABmz57NU089VTWu+r/VDRgwgKysLAoKChg5ciSxsbE8\n9NBDnH/++X6tX0REvHS4QXzi8ssvZ8eOHTWGlZeXO76vLi4ujvT0dJ/WJiIip0chQRoNj8dDUVER\nUVFRREdHB7ocEZEWT4cbJOBKSkoYN248ffr0ITExkZiYGMaNG8+BAwcCXZqISIumkNBEjBkzhlmz\nZgW6DJ9ITp5MZmY2kAYUA2lkZmaTlDQpwJWJiLRsOtwgAeXxeMjIWIM3IKRUDk2hvNySkTGZgoIC\nHXoQEQkQ7UmQgCoqKqp8N7LWmFGA986NIiISGAoJTdD333/PzTffzFlnnUVISAiJiYlVX6YHDx6k\nffv2vP/++zXm+dvf/kZoaChHjhwB4J///Ce/+93vqp7SeO2117Jnzx6/r0tkZGTlu49qjckCICoq\nyq/1iIjIzxQSmqApU6aQm5vLqlWryM7OxlpLYmIi5eXlhIaGMn78eF577bUa87jdbq6//nratm1L\nWVkZCQkJdOrUiQ0bNrBhw4aq5ySUlZX5dV1iYmJISEjE5ZqO95DDl0AaLtcMEhISdahBRCSAFBIC\n7M033+Tiiy+mffv2hIeHM3bsWH744QfHExXfeecdbrzxRt59912WLFlCSkoKq1atokuXLhQUFNCt\nWzdeeuklUlJSePvttzly5AgbN25k4MCBvPnmm2RnZ7Ny5Upat27Njz/+yIsvvkj//v3p06cPS5Ys\nobi4mHXr1vm9D9zuNOLjLwUmAz2AycTHX4rbneb3WkRE5GcKCQH0zTffkJyczO23387OnTvJysri\n+uuvP+HhRtV9//33BAcHM2TIEAAWLFjA8OHD6d+/P4MHD+bOO+8kOjoal8vFihUruOaaa+jYsSPh\n4eE8/fTTzJ07F4Di4uIaT1vs0qULR48erXaOgP+EhYWRnr4aj8fDmjVr8Hg8pKevJiwszO+1iIjI\nz3R1QwB9/fXXlJeXc91119G9e3cALrzwwjotY/z48dxxxx0sXryYUaNGsX37djZs2MCNN97I/Pnz\nCQoKom3btkyaNIlx48Zx7733MnXqVPr378/KlStPCCRdu3ZtsPWrq+joaB1eEBFpRLQnIYAGDhzI\nFVdcwUUXXcRvf/tbXn75Zb7//vtTztO5c2d++uknNm/eDHifd7B//348Hg/9+vWjW7du7Nu3j5SU\nFD777DMiIyPJyspi0iTvPQeO74EoLi6ma9eu9O7du8arY8eOvl1pERFpMhQSAigoKIj333+f9PR0\nLrzwQp599ln69u3L7t27CQoKOuGv/IqKCkJDQ5k4cSJTp07l6NGj7Nu3j0mTJtG9e3cmTpyIMYaK\nigpGjRpFu3bt2LFjB7179yYuLg4Aay3GGDp37szEiRNZv349u3fvZt26dcyYMYN//etfgegKERFp\nhHwaEowxI4wx7xhjvjLGVBhjrvFle03V0KFDefDBB/nkk08IDg7m7bffpmvXrnz99dc1ptu/fz8A\nS5cuJS4ujr179/L0008TFBTE6tWrcblcNaa/7LLLOHToEDfddFPVsK1bt1Yto0ePHtxwww3079+/\nKnSEhob6eG1FRKSp8PU5CSHANuAV4P/3cVtNzpYtW/jggw8YO3YsZ599NtnZ2Xz33Xf069eP9u3b\nM3v2bNasWUNkZCQxMTF88skngPeQw6uvvkpWVhapqalMnz7dcflvvvkmERER7Nmzh507d7Jnzx7m\nz58PQHh4OEuXLvXbuoqISNPj05BgrU0H0gGMMcaXbTVFoaGhfPTRRzzzzDMcPHiQnj17smDBAhIS\nEigrK2P79u1MmTKFVq1akZqayuWXX15jfqcurT6sY8eOrFq1ijvvvJPBgwczYMAAHnzwQZKTk/n6\n66/55z//qScuiojISZlTXW7XoA0ZUwFca6195xTTxAI5OTk5xMbG+qWulmbx4sXcddddVFRUVA1L\nSEjE7U7TJYciIk1Ybm7u8fPP4qy1uQ2xTF0C2cw98cQTBAUFERcXR2lpKTNnzsTaVniPAI0EPiIz\nczpJSZNIT18d4GpFRKQxaZQhITU1lU6dOtUYlpSURFJSUoAqanpKSkpITp5c+YRFr7Zt21Y+u+EV\n9MRFEZGmy+1243a7awwrLS1t8HZ0uKGZGjduPJmZ2ZSXL+L4HoOgoGlUVJQCu4Hu1ab+EujBmjVr\nuOqqqwJQrYiInClfHG7QfRKaIY/HQ0bGmsqAkII3EKRQUbEIqABW1JpDT1wUEZET+fRwgzEmBIgC\njp9y39sYMxAosdZ+6cu2W7Kfn78wstaYUQAEBT1ERcU5lZ+zcLlmEB+vJy6KiEhNvt6T8CvgEyAH\nsMB8IBd42MfttmiRkZGV7z6qNca7x+Cyy+LQExdFROSX+Po+CVnokIbfxcTEkJCQSGbmdMrLLbX3\nGKSnr6agoIDCwkLdJ0FERE6qUV7dIGfO7U4jKWkSGRmTq4bFxydW7THQExdFROSXKCQ0U2FhYdpj\nICIiZ0QhoZnTHgMREakvnS8gIiIijhQSRERExJFCgoiIiDhSSBARERFHCgkiIiLiSCFBREREHCkk\niIiIiCOFBGlRxowZw6xZswJdhohIk6CQICIiIo4UEkRERMSRQoK0OGVlZUybNo3OnTvTtWtXHnjg\ngapxx44dY86cOVxwwQV06NCBoUOHkpWVFcBqRUQCRyFBWpxXX32V4OBgtm7dyqJFi1iwYAFLliwB\n4O6772bz5s2sWLGCTz/9lN/85jdcddVVFBUVBbhqERH/0wOepMXp0aMHCxYsALwPwNq+fTsLFy5k\n7NixvPrqq3z55Zd069YNgFmzZvHee++xdOlSHnvssUCWLSLidwoJ0uJceumlNT4PHTqUBQsW8Omn\nn1JeXk5MTAzW2qrxx44dIzw83N9liogEnEKCSKXDhw/TqlUrcnNzCQqqeSSuQ4cOAapKRCRwFBKk\nxcnOzq4SogofAAAQY0lEQVTxedOmTURHRzN48GDKysrYu3cvw4YNC1B1IiKNh05clBbnyy+/ZM6c\nOXg8HtxuN3/5y1+YOXMmUVFRpKSkcPPNN/PWW2+xe/dutmzZwrx583jvvfcCXbaIiN9pT4K0KMYY\nbr75Zn788UeGDBlCq1atSE1N5fbbbwe8Vz489thjzJkzh6+++oouXbowdOhQJkyYEODKRUT8z1Q/\nQSvQjDGxQE5OTg6xsbGBLkdERKTJyM3NJS4uDiDOWpvbEMvUngSRajweD0VFRURFRREdHR3ockRE\nAkrnJIgAJSUljBs3nj59+pCYmEhMTAzjxo3nwIEDgS5NRCRgFBKk0cnKyiIoKIiDBw/6rc3k5Mlk\nZmYDaUAxkEZmZjZJSZP8VoOISGOjkCAB5/T4ZmOM39r3eDxkZKyhvHwRkAJ0B1IoL3+GjIw1FBQU\n+K0WEZHGRCFBWryfn8swstaYUQAUFhb6tR4RkcZCIUEC6pZbbiErK4tnnnmGoKAgXC4Xu3fvBuDj\njz/mkksuISQkhGHDhp3wF/3KlSuJi4ujXbt2REVF8cgjj1BRUVHnGiIjIyvffVRrjPfpj1FRUXVe\npohIc6CQIAH1zDPPMHToUKZOncrevXv5+uuv6d69O9Za/vjHP7Jw4UJycnJo1aoVt956a9V869ev\nZ8qUKaSmprJz504WL17MsmXL+POf/1znGmJiYkhISMTlmo73nIQvgTRcrhkkJCTqKgcRabEUEiSg\nQkNDad26Ne3bt6dr166cffbZuFwujDH813/9F8OHD6dv377cd999bNy4kWPHjgHw8MMPc//99zNp\n0iR69uzJFVdcwSOPPMILL7xQrzrc7jTi4y8FJgM9gMnEx1+K253WYOsqItLU6D4J0mgNGDCg6v25\n554LwL59+7jgggvIy8tj48aNNR7fXF5ezrFjxzhy5Aht27atU1thYWGkp6+moKCAwsJC3SdBRASF\nBGnEgoODq94fv9rh+DkHhw4d4pFHHuH6668/Yb66BoTqoqOjFQ5ERCopJEjAtW7dmvLy8jrNExsb\nyz/+8Q969+7to6pEREQhQQKuV69ebN68mT179tChQwcqKipweqZI9WEPPPAAEyZMoHv37tx4440E\nBQWRl5fHjh07ePTRR/1ZvohIs6UTFyXg5syZg8vlon///px99tkUFxc73kyp+rCxY8eyatUq/v73\nvzNkyBCGDh3K008/Ta9evfxYuYhI86anQIqIiDQDvngKpPYkiIiIiCOdkyBNlh7rLCLiW9qTIE2O\nHussIuIfCgnS5OixziIi/uGXkGCMudsY84Ux5kdjTLYx5hJ/tCvNjx7rLCLiPz4PCcaY3wHzgQeB\nwUAekGGMCfd129L86LHOIiL+4489CanAYmvtcmvtTuAO4Afg1lPPJnIiPdZZRMR/fBoSjDHBQBzw\nwfFh1ntjhkxgqC/bluZJj3UWEfEfX+9JCAdcwN5aw/cC3XzctjRTeqyziIh/BOo+CQZoPLd6lCZF\nj3UWEfEPX4eE74By4Jxaw8/mxL0LVVJTU+nUqVONYUlJSSQlJTV4gdJ06bHOItJSud1u3G53jWGl\npaUN3o7Pn91gjMkGNltrZ1R+Nngvbl9krX2y1rR6doOIiEg9+OLZDf443LAAWGaMyQG24L3aoT3w\nqh/aFhERkXryeUiw1q6ovCfCI3gPO2wDEqy13/q6bREREak/v5y4aK19DnjOH22JiIhIw9CzG0RE\nRMSRQoKIiIg4UkgQERERRwoJIiIi4kghQURERBwpJIiIiIgjhQQRERFxpJAgIiIijhQSRERExJFC\ngoiIiDhSSBARERFHCgkiIiLiSCFBREREHCkkiIiIiCOFBBEREXGkkCAiIj6zbNkyzjrrrDrNM2bM\nGGbNmuWjiqQuWgW6ABERab5uuukmxo8fX6d53nrrLYKDg31UkdSFQoKIiNTLTz/99Itf5m3atKFN\nmzZ1Wm7nzp3PpCxpQDrcICIip2XMmDFMmzaN1NRUunbtyrhx41i4cCEXX3wxHTp0oEePHtx9990c\nPny4ap5ly5YRFhZW9fnhhx9m8ODBpKWlERERQefOnUlKSqoxT+3DDRERETz++OPcdttthIaG0rNn\nT1566aUatW3cuJHBgwfTrl07hgwZwsqVKwkKCmL79u0+7JHmTyFBRERO2/Lly2nTpg0bN27khRde\nwOVy8eyzz/LZZ5+xfPlyPvzwQ+bOnVtjHmNMjc9FRUWsXLmSNWvWsHr1arKyspg3b94p212wYAGX\nXHIJ27Zt46677uLOO+/E4/EAcOjQIa655hoGDhzIJ598wqOPPsrcuXNPaFfqTiFBREROW1RUFPPm\nzSM6Opro6GimT5/OqFGj6NmzJ6NHj+bRRx9lxYoVp1yGtZZly5bRr18/hg0bxuTJk/nggw9OOc/4\n8eO544476N27N3PnziU8PJx169YBkJaWRlBQEC+++CJ9+/YlISGBe++9t6FWuUXTOQkiInLafvWr\nX9X4nJmZybx589i5cycHDx6krKyMo0eP8uOPP9KuXTvHZfTq1Yv27dtXfT733HPZt2/fKdsdMGBA\njc/dunWrmsfj8XDxxRfTunXrqvFDhgyp03qJM+1JEBGR0xYSElL1fs+ePUyYMIFBgwbxt7/9jdzc\nXP7nf/4H8J7UeDK1T3Y0xlBRUXHKdk81j7X2hEML1tpfXhn5RQoJIiJSLzk5OVRUVPDUU08xZMgQ\noqKi+Oqrr/xeR9++fdm+fXuNYLJ161a/19EcKSSIiEi9REVFUVZWxqJFi/jiiy/461//yuLFi/1e\nR3JyMuXl5UydOpWdO3eSkZHB/PnzgRNPmpS6UUgQEZHTUvsL9+KLL2bBggU88cQTDBgwALfb/YtX\nKdSnHacv+urDOnbsyKpVq8jLy2Pw4MH86U9/4sEHHwSgbdu2Z1xPS2Ya03EbY0wskJOTk0NsbGyg\nyxERkSbqtdde47bbbqO0tLTON3NqqnJzc4mLiwOIs9bmNsQydXWDiIg0eU888QRBQUHExcVRWlrK\nfffdx+9+97sWExB8RSFBRESarJKSEpKTJ5ORsaZqWLt27bn11lt46qmnAlhZ86BzEkREpMlKTp5M\nZmY2kAYUA2kcO9aWwsIvdD5CA9CeBBERaZI8Hk/lHoQ0IKVyaArl5ZaMjMkUFBQQHR0dwAqbPu1J\nEBGRJqmoqKjy3chaY0YBUFhY6Nd6miOFBBERaZIiIyMr331Ua0wW4L2Pg5wZhQQREWmSYmJiSEhI\nxOWajveQw5dAGi7XDBISEnWooQEoJIiISJPldqcRH38pMBnoAUwmPv5S3O60AFfWPOjERRERabLC\nwsJIT19NQUEBhYWFREVFaQ9CA1JIEBGRJi86OlrhwAd0uEFEREQcKSSIiIiII4UEERERcaSQ0Ey4\n3e5Al9DkqM/qR/1Wd+qz+lG/BZ7PQoIx5v8aYzYYYw4bY0p81Y546Yep7tRn9aN+qzv1Wf2o3wLP\nl3sSgoEVwPM+bENERER8xGeXQFprHwYwxkzxVRsiIiLiOzonQURERBw1tpsptQXIz88PdB1NTmlp\nKbm5uYEuo0lRn9WP+q3u1Gf1o36rm2rfnW0bapnGWnv6ExvzODD3FJNYoJ+11lNtninAQmvtWaex\n/GTgtdMuSERERGpLsda+3hALquuehKeApb8wza561gKQAaQAu4EjZ7AcERGRlqYt0Avvd2mDqFNI\nsNbuB/Y3VOMnWX6DpB8REZEWaGNDLsxn5yQYY7oDZwE9AZcxZmDlqEJr7WFftSsiIiINo07nJNRp\nwcYsBW52GDXGWvuRTxoVERGRBuOzkCAiIiJNm+6TICIiIo4UEkRERMSR30OCMeZuY8wXxpgfjTHZ\nxphLTjHtFGNMhTGmvPLfCmPMD/6sN9CMMSOMMe8YY76qXP9rTmOe0caYHGPMEWOMpyXeGruu/WaM\nGVVtG6uott2d7a+aA80Yc78xZosx5qAxZq8x5i1jTMxpzPcbY0x+5c90njHmKn/U2xjUp8/0ew2M\nMXdUbiulla+NxphxvzBPi93OoO591lDbmV9DgjHmd8B84EFgMJAHZBhjwk8xWynQrdqrp6/rbGRC\ngG3A3XhvVnVKxphewCrgA2Ag8AzwsjHmSt+V2CjVqd8qWSCan7e1c621+3xTXqM0AngW+DUQj/ch\nbe8bY9qdbAZjzFC8ly2/BAwC3gbeNsb09325jUKd+6xSS/+99iXeG/PFVb7WAiuNMf2cJtZ2BtSx\nzyqd+XZmrfXbC8gGnqn22QD/BP5wkumnACX+rLExv4AK4JpfmOa/ge21hrmBNYGuv5H32yigHAgN\ndL2N5QWEV/bd8FNM8/8B79Qatgl4LtD1N+I+0+81537ZD9xyknHazureZw2ynfltT4IxJhhv+vng\n+DDrXZNMYOgpZu1gjNltjCk2xrS05Fgfl+Lt0+oyOHUfi5cBthlj/mWMed8Yc1mgCwqwznj3rpSc\nYpqhaHur7nT6DPR7rYoxJsgYcxPQHu8XvxNtZ9WcZp9BA2xn/jzcEA64gL21hu/FuxvEyT+AW4Fr\n8N6uOQjYaIw531dFNgPdcO7jUGNMmwDU01R8DfweuAG4Hu+uvXXGmEEBrSpAjDEGeBpYb639/BST\nnmx7O9nPdLNVhz7T7zXAGHORMebfwFHgOeA6a+3Ok0yu7Yw691mDbGeN4SmQhpMcM7bWZuM9ROGd\n0JhNQD7wf/Ce1yCnx1T+q5tinIT1PpTMU21QtjEmEkjFu9uupXkO6A8Mq8e8J/2ZbuZOq8/0e63K\nTrznTXXGG86XG2NGnuJLr7aWuJ2ddp811Hbmz5DwHd5jvufUGn42JyZER9baMmPMJ0BUA9fWnHyD\ncx8ftNYeC0A9TdkW6vcl2aQZY/4CJAIjrLVf/8LkJ9veTutnurmoY5/V0FJ/r1lry/j5gYC5xpgh\nwAzgTofJtZ1R5z47Yd76bGd+O9xgrf0JyAGuOD6scvfcFZzmAymMMUHARXh3DYuzTVTr40pjOfVx\nK3E2iBa2rVV+2U3Ee/v04tOYxWl7u5IWtL3Vo89qz6/fa15BwMkOibb47ewkTtVnNdR7O/PzmZi/\nBX7E+0yHvsBivGdndq0cvxz4r2rT/wnvhhCB95JJN3AY6Bvos0r92GcheHcvDcJ71vTMys/dK8c/\nDiyrNn0v4BDeqxz6AHcBx4D4QK9LI++3GXiP3UUCF+I9tvwTMDrQ6+LHPnsOOID3sr5zqr3aVptm\nWa2f0aGV29esyu3tIbyPee8f6PVpxH2m32vwZ2A43kvyLqr8eSwDLq8cX/u7oEVvZ/XsswbZzgKx\noncBu/GGhU3Ar6qNWwu8Uu3zAuCLymn/BbwLXBzo/yw/99eoyi+58lqvVyrHLwXWOsyTU9lvBcDk\nQK9HY+834N7KvjoMfIv3KpyRgV4PP/eZU3+VAzdXm6bGz2jlsBvwHiv9EdgOJAR6XRpzn+n3mgV4\nGe9u8x/xHkp4//iXnbazhumzhtrO9IAnERERcaRnN4iIiIgjhQQRERFxpJAgIiIijhQSRERExJFC\ngoiIiDhSSBARERFHCgkiIiLiSCFBREREHCkkiIiIiCOFBBEREXGkkCAiIiKO/h/w1QYloCQ6cwAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x135472590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Launch the graph in a session\n",
    "with tf.Session() as sess:\n",
    "    # Initializing all variables\n",
    "    tf.global_variables_initializer().run()\n",
    "\n",
    "    for step in range(100):\n",
    "        batch_inputs, batch_labels = generate_batch(batch_size)\n",
    "        _, loss_val = sess.run([train_op, loss],\n",
    "                feed_dict={train_inputs: batch_inputs, train_labels: batch_labels})\n",
    "        if step % 10 == 0:\n",
    "          print(\"Loss at \", step, loss_val) # Report the loss\n",
    "\n",
    "    # Final embeddings are ready for you to use. Need to normalize for practical use\n",
    "    trained_embeddings = embeddings.eval()\n",
    "\n",
    "# Show word2vec if dim is 2\n",
    "if trained_embeddings.shape[1] == 2:\n",
    "    labels = rdic[:10] # Show top 10 words\n",
    "    for i, label in enumerate(labels):\n",
    "        x, y = trained_embeddings[i,:]\n",
    "        plt.scatter(x, y)\n",
    "        plt.annotate(label, xy=(x, y), xytext=(5, 2),\n",
    "            textcoords='offset points', ha='right', va='bottom')\n",
    "    plt.savefig(\"word2vec.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}