initial commit

Author: alonecoder1337

App.py

@@ -0,0 +1,50 @@
+from Classifier import Classififer
+import pandas as pd
+import numpy as np
+from Dataset import Dataset
+
+
+class App:
+    def __init__(self):
+        self.classifier = Classififer().get_classifier();
+
+
+    def train(self):
+        df = pd.read_csv('data/train.csv', header=None)
+        data = np.array(df)
+        self.x_train = data[:, :-1]
+        self.y_train = data[:, -1:]
+        self.classifier.fit(self.x_train,self.y_train)
+
+    def test(self):
+        self.ds_obj = Dataset()
+        ds = self.ds_obj.read_dataset()
+        new_ds = []
+        for row in ds:
+            new_ds.append(row[1:])
+        self.x_test = np.array(new_ds)
+        self.results = self.classifier.predict(self.x_test)
+
+    def post_test(self):
+        client_ip_ids = []
+        total_test,_ = self.x_test.shape
+
+        for i in range(total_test):
+            if self.results[i]==1 :
+                if self.x_test[i,1] not in client_ip_ids:
+                    client_ip_ids.append(self.x_test[i,1])
+        dos_ips = self.ds_obj.detransform_client_ip(np.array(client_ip_ids,dtype="int64"))
+        for ip in dos_ips:
+            print ip
+
+    def run(self):
+        self.train()
+        self.test()
+        self.post_test()
+
+
+if __name__ == '__main__':
+    app = App()
+    app.run()
+
+

Classifier.py

@@ -0,0 +1,13 @@
+from sklearn import svm
+from sklearn.tree import DecisionTreeClassifier
+
+class Classififer:
+    def __init__(self):
+        pass
+
+    def get_classifier(self):
+        '''
+            returns Classifier object
+        '''
+        clf = DecisionTreeClassifier()
+        return clf

Classifier.pyc

@@ -0,0 +1,115 @@
+import sklearn
+import numpy as np
+from utils.LogHelper import Logs
+from utils.DateUtil import get_microseconds
+from utils.Anomaly import Anomaly
+from sklearn import preprocessing
+
+
+class Dataset:
+
+    def __init__(self):
+
+        self.logs = Logs().read()
+        self.client_ip_label_encoder = preprocessing.LabelEncoder()
+        self.request_method_label_encoder = preprocessing.LabelEncoder()
+        self.request_status_label_encoder = preprocessing.LabelEncoder()
+        self.request_size_label_encoder = preprocessing.LabelEncoder()
+        self.time_taken_to_serve_label_encoder =preprocessing.LabelEncoder()
+        self.user_agent_label_encoder =preprocessing.LabelEncoder()
+        self.request_header_label_encoder = preprocessing.LabelEncoder()
+
+        self.scores = []
+        self.client_ips = []
+        self.request_methods = []
+        self.request_status = []
+        self.request_size = []
+        self.times_taken_to_serve = []
+        self.user_agents = []
+        self.request_headers = []
+
+        self.dataset = []
+
+    def preprocess_time(self):
+        timestamp_clusters = {}
+        for row in self.logs:
+            timestamp = get_microseconds(row[0])
+            if timestamp not in timestamp_clusters:
+                timestamp_clusters[timestamp]=0
+            timestamp_clusters[timestamp] = timestamp_clusters[timestamp] + 1
+        anomaly_scores = Anomaly().detect(timestamp_clusters)
+        for row in self.logs:
+            self.scores.append(anomaly_scores[row[0]])
+
+    def preprocess_client_ip(self):
+        self.client_ip_label_encoder.fit([row[1] for row in self.logs])
+        inst = [row[1] for row in self.logs]
+        self.client_ips = self.client_ip_label_encoder.transform(inst)
+
+    def preprocess_request_method(self):
+        self.request_method_label_encoder.fit([row[2] for row in self.logs])
+        inst = [row[2] for row in self.logs]
+        self.request_methods = self.request_method_label_encoder.transform(inst)
+
+    def preprocess_request_status(self):
+        self.request_status_label_encoder.fit([row[3] for row in self.logs])
+        inst = [row[3] for row in self.logs]
+        self.request_status = self.request_status_label_encoder.transform(inst)
+
+    def preprocess_request_size(self):
+        self.request_size_label_encoder.fit([row[4] for row in self.logs])
+        inst = [row[4] for row in self.logs]
+        self.request_size = self.request_size_label_encoder.transform(inst)
+
+    def preprocess_time_taken_to_serve(self):
+        self.time_taken_to_serve_label_encoder.fit([row[5] for row in self.logs])
+        inst = [row[5] for row in self.logs]
+        self.times_taken_to_serve = self.time_taken_to_serve_label_encoder.transform(inst)
+
+    def proprocess_user_agent(self):
+        self.user_agent_label_encoder.fit([row[6] for row in self.logs])
+        inst = [row[6] for row in self.logs]
+        self.user_agents = self.user_agent_label_encoder.transform(inst)
+
+    def preprocess_request_header(self):
+        self.request_header_label_encoder.fit([row[7] for row in self.logs])
+        inst = [row[7] for row in self.logs]
+        self.request_headers = self.request_header_label_encoder.transform(inst)
+
+    def detransform_client_ip(self, client_ip_list):
+        return self.client_ip_label_encoder.inverse_transform(client_ip_list)
+
+    def preprocess(self):
+
+        self.preprocess_time()
+        self.preprocess_client_ip()
+        self.preprocess_request_method()
+        self.preprocess_request_status()
+        self.preprocess_request_size()
+        self.preprocess_time_taken_to_serve()
+        self.proprocess_user_agent()
+        self.preprocess_request_header()
+
+        dataset_size = len(self.logs)
+        for i in range(dataset_size):
+            obj = [
+                self.logs[i][0],
+                self.scores[i],
+                self.client_ips[i],
+                self.request_methods[i],
+                self.request_status[i],
+                self.request_size[i],
+                self.times_taken_to_serve[i],
+                self.user_agents[i],
+                self.request_headers[i]
+            ]
+            self.dataset.append(obj)
+
+    def read_dataset(self):
+        self.preprocess()
+        return self.dataset
+
+
+if __name__=='__main__':
+    dataset_obj = Dataset()
+    dataset_obj.preprocess()

Dataset.py

@@ -0,0 +1,32 @@
+<VirtualHost *:80>
+	# The ServerName directive sets the request scheme, hostname and port that
+	# the server uses to identify itself. This is used when creating
+	# redirection URLs. In the context of virtual hosts, the ServerName
+	# specifies what hostname must appear in the request's Host: header to
+	# match this virtual host. For the default virtual host (this file) this
+	# value is not decisive as it is used as a last resort host regardless.
+	# However, you must set it for any further virtual host explicitly.
+	#ServerName www.example.com
+
+	ServerAdmin webmaster@localhost
+	DocumentRoot /var/www/html
+
+	# Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
+	# error, crit, alert, emerg.
+	# It is also possible to configure the loglevel for particular
+	# modules, e.g.
+	#LogLevel info ssl:warn
+
+	ErrorLog ${APACHE_LOG_DIR}/error.log
+	#CustomLog ${APACHE_LOG_DIR}/access.log combined
+	CustomLog ${APACHE_LOG_DIR}/custom.log dos
+	
+	# For most configuration files from conf-available/, which are
+	# enabled or disabled at a global level, it is possible to
+	# include a line for only one particular virtual host. For example the
+	# following line enables the CGI configuration for this host only
+	# after it has been globally disabled with "a2disconf".
+	#Include conf-available/serve-cgi-bin.conf
+</VirtualHost>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet