comit

Author: ram1988

lstm_network.py

@@ -0,0 +1,293 @@
+
+import pandas as pd
+import tensorflow as tf
+import numpy as np
+
+from tensorflow.contrib import rnn
+
+
+class LSTMNet:
+    start = 0
+    model_file = "./author_lstm.model"
+
+    def __init__(self, nfeatures, max_length, vocab_size, embedding_matrix):
+        self.nfeatures = nfeatures
+        self.n_hidden = nfeatures / 2
+        self.n_steps = max_length
+        self.n_layers = 1
+        self.batch_size = 200
+        self.dropout = 0.7
+        self.max_length = max_length
+        self.embedding_matrix = embedding_matrix
+        self.vocab_size = vocab_size
+        self.threshold = 0.7
+        self.learning_rate = 0.08
+        self.epsilon = 1e-3
+        self.istraining = True
+
+    def __createBatch(self, input1=None, labels=None, batch_size=None):
+
+        self.end = self.start + batch_size
+
+        batch_x1 = input1[self.start:self.end]
+        print(len(batch_x1))
+
+        batch_y = labels[self.start:self.end]
+
+        self.start = self.end
+
+        if (self.end >= len(input1)):
+            self.start = 0
+
+        return batch_x1, batch_y
+
+    def __createTestBatch(self, input1=None, batch_size=None):
+
+        self.end = self.start + batch_size
+
+        batch_x1 = input1[self.start:self.end]
+        print(len(batch_x1))
+
+        self.start = self.end
+
+        if (self.end >= len(input1)):
+            self.start = 0
+
+        return batch_x1
+
+    def convertLabelsToOneHotVectors(self, labels):
+
+        one_hot_label = []
+
+        for label in labels:
+            if label == 0:
+                one_hot_label.append([1, 0, 0])
+            elif label == 1:
+                one_hot_label.append([0, 1, 0])
+            else:
+                one_hot_label.append([0, 0, 1])
+
+        return one_hot_label
+
+    def reshape(self, input1, labels=None):
+        input1 = np.reshape(input1, (-1, self.max_length))
+        labels = np.reshape(labels, (-1, 1))
+
+        return input1, labels
+
+    def insertBatchNNLayer(self, mat_rel, axes, dimension_size):
+        mean = None
+        var = None
+        batch_mean, batch_var = tf.nn.moments(mat_rel, axes)
+        ema = tf.train.ExponentialMovingAverage(decay=0.5)
+
+        if self.istraining:
+            print("is training in BN")
+            mean = batch_mean
+            var = batch_var
+            ema_apply_op = ema.apply([batch_mean, batch_var])
+        else:
+            print("is testing in BN")
+            mean = ema.average(batch_mean)
+            var = ema.average(batch_var)
+
+        scale2 = tf.Variable(tf.ones(dimension_size, dtype=tf.float64), dtype=tf.float64)
+        beta2 = tf.Variable(tf.zeros(dimension_size, dtype=tf.float64), dtype=tf.float64)
+        bn_layer = tf.nn.batch_normalization(mat_rel, mean, var, beta2, scale2, self.epsilon)
+
+        return bn_layer
+
+    #Build 6 softmax layers to predict the sentiments
+    def buildRNN(self, x, scope):
+        print(x)
+        x = tf.transpose(x, [1, 0, 2])
+
+
+        with tf.name_scope("fw" + scope), tf.variable_scope("fw" + scope):
+            fw_cell_array = []
+            print(tf.get_variable_scope().name)
+            for _ in range(self.n_layers):
+                fw_cell = rnn.GRUCell(self.n_hidden, activation=tf.nn.relu)
+                fw_cell = rnn.DropoutWrapper(fw_cell,input_keep_prob=self.dropout,output_keep_prob=self.dropout)
+                fw_cell_array.append(fw_cell)
+            fw_cell = rnn.MultiRNNCell(fw_cell_array, state_is_tuple=True)
+        with tf.name_scope("bw" + scope), tf.variable_scope("bw" + scope):
+            bw_cell_array = []
+            print(tf.get_variable_scope().name)
+            for _ in range(self.n_layers):
+                bw_cell = rnn.GRUCell(self.n_hidden, activation=tf.nn.relu)
+                bw_cell = rnn.DropoutWrapper(bw_cell,input_keep_prob=self.dropout,output_keep_prob=self.dropout)
+                bw_cell_array.append(bw_cell)
+            bw_cell = rnn.MultiRNNCell(bw_cell_array, state_is_tuple=True)
+
+
+        outputs = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, x, dtype=tf.float64, time_major=True)
+        print("output-->" + str(outputs))
+        outputs = tf.concat(outputs[0], 2)
+        outputs = tf.reshape(outputs, [-1, self.nfeatures])
+        outputs = tf.split(outputs, self.n_steps, 0)
+        print("output-->"+str(outputs))
+        outputs = outputs[-1]
+        print("output-->" + str(outputs))
+
+        nn_layer1 = tf.layers.dense(outputs,1024,activation=tf.nn.relu)
+        nn_layer1 = tf.layers.dropout(nn_layer1,rate=0.8)
+        nn_layer2 = tf.layers.dense(nn_layer1, 1024, activation=tf.nn.relu)
+        nn_layer2 = tf.layers.dropout(nn_layer2, rate=0.8)
+        result = tf.layers.dense(nn_layer2, 3, activation=tf.nn.softmax)
+
+        print("final result11-->"+str(result))
+
+        return result
+
+    def optimizeWeights(self, pred):
+        #cost = tf.reduce_mean(-tf.reduce_sum(self.y * tf.log(pred), reduction_indices=1))
+        print("predicted-->"+str(pred))
+        cost = tf.losses.log_loss(self.y, pred)
+        #global_step = tf.Variable(0, trainable=False)
+        #learning_rate = tf.train.exponential_decay(self.learning_rate, global_step,
+        #                                           1000, 0.5, staircase=False)
+        optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(cost)
+
+        return optimizer, cost
+
+
+    def prepareFeatures(self):
+        x1 = tf.placeholder(tf.int32, [None, self.max_length])  # batch_size x sentence_length
+        y = tf.placeholder(tf.float64, [None, 3], "labels")
+
+        return x1, y
+
+
+    #connect to fully connected layer of 3 target classes
+    def trainModel(self, input1, labels, one_hot_encoding=False):
+        # Parameters
+
+        training_epochs = 10
+        display_step = 1
+        record_size = len(input1)
+        labels = self.convertLabelsToOneHotVectors(labels)
+
+
+        self.x1, self.y = self.prepareFeatures()
+        self.embedded_chars1 = tf.nn.embedding_lookup(self.embedding_matrix, self.x1,
+                                                      name="lookup1")  # batch_size x sent_length x embedding_size
+
+        print("Embedding-->" + str(self.embedded_chars1))
+        print("Embedding-->" + str(self.x1))
+
+
+        self.pred = self.buildRNN(self.embedded_chars1, "nn1_side")
+
+
+        # Initializing the variables
+        optimizer, cost = self.optimizeWeights(self.pred)
+        init = tf.global_variables_initializer()
+
+        with tf.Session() as sess:
+            sess.run(init)
+            count = 0
+            labels = np.reshape(labels, (-1, 3))
+
+            # Training cycle
+            # Change code accordingly
+            for epoch in range(training_epochs):
+                print("Epoch--->" + str(epoch))
+                avg_cost = 0.
+                total_batch = int(record_size / self.batch_size)
+                # Loop over all batches
+                for i in range(total_batch):
+                    print("batch--->" + str(i))
+                    batch_x1, batch_ys = self.__createBatch(input1, labels, self.batch_size)
+
+
+                    # Run optimization op (backprop) and cost op (to get loss value)
+                    _, c = sess.run([optimizer, cost], feed_dict={self.x1: batch_x1, self.y: batch_ys})
+                    print("cost per step-->"+str(c))
+                    # Compute average loss
+                    avg_cost += c / total_batch
+                    count = count + self.batch_size
+                # Display logs per epoch step
+                if (epoch + 1) % display_step == 0:
+                    # -1304 cost :0
+                    print("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(avg_cost))
+
+            #saver.save(sess, self.model_file)
+
+        print("Optimization Finished!")
+
+    def validateModel(self, test_input1, test_labels, one_hot_encoding=False):
+
+        self.istraining = False
+        test_labels = self.convertLabelsToOneHotVectors(test_labels)
+
+        test_input1 = np.asarray(test_input1)
+        test_labels = np.asarray(test_labels)
+
+        print("Test1--->" + str(len(test_input1)))
+
+        #test_input1, test_labels = self.reshape(test_input1, test_labels)
+
+
+
+        print(len(test_input1))
+        print(len(test_labels))
+        record_size = len(test_input1)
+
+        init_op = tf.global_variables_initializer()
+        #saver = tf.train.Saver()
+
+        with tf.Session() as sess:
+            sess.run(init_op)
+            #saver.restore(sess, self.model_file)
+            overall_accuracy = 0
+
+            total_batch = int(record_size / self.batch_size)
+            for i in range(total_batch):
+                batch_x1, batch_ys = self.__createBatch(test_input1,  test_labels,
+                                                                  self.batch_size)
+                print(len(batch_x1))
+                predictions = sess.run([self.pred], feed_dict={self.x1: batch_x1})
+                # Compute Accuracy
+                batch_log_loss = tf.losses.log_loss(predictions[0], batch_ys)
+                print("Log Loss:", batch_log_loss.eval())
+
+
+    def evaluateResults(self, predictions, actual):
+        print(predictions)
+        predictions = predictions[0]
+        predicted = tf.equal(tf.argmax(predictions, 1), tf.argmax(actual, 1))
+        batch_accuracy = tf.reduce_mean(tf.cast(predicted, "float"), name="accuracy")
+        batch_accuracy = batch_accuracy.eval()
+
+        return batch_accuracy
+
+    def predict(self, test_input1):
+        # Test model
+        self.istraining = False
+        result = []
+        test_input1 = np.asarray(test_input1)
+        record_size = len(test_input1)
+
+
+        init = tf.global_variables_initializer()
+        with tf.Session() as sess:
+            sess.run(init)
+
+            total_batch = int(record_size / self.batch_size) + 1
+
+            for i in range(total_batch):
+                batch_x1 = self.__createTestBatch(test_input1, batch_size=self.batch_size)
+
+                print(len(batch_x1))
+                predictions = sess.run([self.pred], feed_dict={self.x1: batch_x1})
+                print(predictions)
+                result.extend(predictions[0])
+                # print(result)
+
+        return result
+
+    def generatePrediction(self,predictions):
+        predicted = tf.argmax(predictions[0],1)
+        predicted = predicted.eval()
+        return predicted