Improve package structure + logging & added commandline import option.

Author: Robin Duda

README.md

@@ -10,7 +10,7 @@ ElasticSearch and Kibana (version 5.0.0) should not require any additional confi
 ## Running
 Running the application,
 ```
-java -jar excelastic-1.0.1.jar run Launcher
+java -jar excelastic-1.0.2.jar
 ```
 
 If any connection errors occur check that the ElasticSearch listen port matches with the elastic_port in the configuration file. Make sure that ElasticSearch is running by directing your browser at [localhost:9200](http://localhost:9200/_count). When the server is started browse to [localhost:8080](http://localhost:8080/) for the web interface.

pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>com.codingchili</groupId>
     <artifactId>excelastic</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.2</version>
     <build>
         <plugins>
             <plugin>
@@ -37,7 +37,7 @@
                             <transformers>
                                 <transformer
                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>io.vertx.core.Launcher</mainClass>
+                                    <mainClass>com.codingchili.Launcher</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>
@@ -50,32 +50,32 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>RELEASE</version>
+            <version>4.12</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>io.vertx</groupId>
             <artifactId>vertx-core</artifactId>
-            <version>3.3.3</version>
+            <version>3.4.1</version>
         </dependency>
 
         <dependency>
             <groupId>io.vertx</groupId>
             <artifactId>vertx-unit</artifactId>
-            <version>3.3.3</version>
+            <version>3.4.1</version>
         </dependency>
 
         <dependency>
             <groupId>io.vertx</groupId>
             <artifactId>vertx-web</artifactId>
-            <version>3.3.3</version>
+            <version>3.4.1</version>
         </dependency>
 
         <dependency>
             <groupId>io.vertx</groupId>
             <artifactId>vertx-web-templ-jade</artifactId>
-            <version>3.3.3</version>
+            <version>3.4.1</version>
         </dependency>
 
         <dependency>

src/main/java/Launcher.java

@@ -1,6 +1,6 @@
-package Controller;
+package com.codingchili.Controller;
 
-import Model.*;
+import com.codingchili.Model.*;
 import io.vertx.core.*;
 import io.vertx.core.buffer.Buffer;
 import io.vertx.core.json.JsonObject;
@@ -12,17 +12,19 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.Iterator;
+import java.util.logging.Logger;
 
 /**
  * @author Robin Duda
+ *
+ * Manages the web interface and handles file uploads.
  */
 public class Website extends AbstractVerticle {
+    private Logger logger = Logger.getLogger(getClass().getName());
     private static final String DONE = "/done";
     private static final String ERROR = "/error";
     private static final String MESSAGE = "message";
     private static final String OFFSET = "offset";
-    private static final String INDEX = "index";
-    private static final String ITEMS = "items";
     private static final String FILE = "file";
     private static final String NO_FILE_WAS_UPLOADED = "No file was uploaded.";
     private Vertx vertx;
@@ -43,8 +45,14 @@ public void start(Future<Void> start) {
         router.route("/static/*").handler(StaticHandler.create());
         router.route("/*").handler(TemplateHandler.create(engine));
 
-        vertx.createHttpServer().requestHandler(router::accept).listen(Configuration.WEB_PORT);
-        start.complete();
+        vertx.createHttpServer().requestHandler(router::accept).listen(Configuration.WEB_PORT, done -> {
+            if (done.succeeded()) {
+                logger.info("Started website on port " + Configuration.WEB_PORT);
+                start.complete();
+            } else {
+                start.fail(done.cause());
+            }
+        });
     }
 
     private void setRouterAPI(Router router) {
@@ -80,20 +88,14 @@ private String traceToText(Throwable throwable) {
     private void parse(Buffer buffer, MultiMap params) throws ParserException {
         try {
             int columnRow = Integer.parseInt(params.get(OFFSET));
-            String index = params.get(INDEX);
             FileParser parser = new FileParser(buffer.getBytes(), columnRow);
-
-            JsonObject result = new JsonObject()
-                    .put(ITEMS, parser.toJsonArray())
-                    .put(INDEX, index.toLowerCase());
-
-            sendOutput(result);
+            sendOutput(parser.toImportableObject());
         } catch (NumberFormatException e) {
             throw new ParserException(e);
         }
     }
 
     private void sendOutput(JsonObject data) {
-        vertx.eventBus().send(Configuration.BUS_TRANSACTIONS, data);
+        vertx.eventBus().send(Configuration.INDEXING_ELASTICSEARCH, data);
     }
 }

src/main/java/Controller/Website.java renamed to src/main/java/com/codingchili/Controller/Website.java

@@ -0,0 +1,87 @@
+package com.codingchili;
+
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.codingchili.Controller.Website;
+import com.codingchili.Model.Configuration;
+import com.codingchili.Model.ElasticWriter;
+import com.codingchili.Model.FileParser;
+import com.codingchili.Model.ParserException;
+
+import io.vertx.core.*;
+
+/**
+ * @author Robin Duda
+ *
+ * Launcher class to bootstrap the application.
+ */
+public class Launcher {
+    private static final Logger logger = Logger.getLogger(Launcher.class.getName());
+    private static final String COMMAND_IMPORT = "import";
+    private Vertx vertx = Vertx.vertx();
+
+    public static void main(String[] args) {
+        new Launcher(args);
+    }
+
+    public Launcher(String[] args) {
+        logger.info("Starting application..");
+        Future<Void> future = Future.future();
+
+        future.setHandler(done -> {
+            if (done.succeeded()) {
+                logger.info("Successfully started application");
+                getImportFileName(args).ifPresent(this::importFile);
+            } else {
+                logger.log(Level.SEVERE, "Failed to start application", done.cause());
+            }
+        });
+
+        start(future);
+    }
+
+    private void importFile(String fileName) {
+        logger.info(String.format("Loading file %s from filesystem..", fileName));
+        vertx.fileSystem().readFile(fileName, file -> {
+            if (file.succeeded()) {
+                logger.info("Parsing XLSX file to JSON..");
+                try {
+                    FileParser parser = new FileParser(file.result().getBytes(), 1);
+                    logger.info("File parsed, starting import to elasticsearch..");
+                    vertx.eventBus().send(Configuration.INDEXING_ELASTICSEARCH, parser.toImportableObject());
+                } catch (ParserException e) {
+                    logger.log(Level.SEVERE, String.format("Failed to import file %s", fileName));
+                }
+            } else {
+                logger.log(Level.SEVERE, String.format("Failed to load file %s", fileName), file.cause());
+            }
+        });
+    }
+
+    private static Optional<String> getImportFileName(String[] args) {
+        for (int i = 0; i < args.length; i++) {
+            switch (args[i]) {
+                case COMMAND_IMPORT:
+                    return Optional.of(args[++i]);
+            }
+        }
+        return Optional.empty();
+    }
+
+    public void start(Future<Void> start) {
+        Future<String> writer = Future.future();
+        Future<String> website = Future.future();
+        vertx.deployVerticle(new ElasticWriter(), writer.completer());
+        vertx.deployVerticle(new Website(), website.completer());
+
+        CompositeFuture.all(writer, website).setHandler(done -> {
+            if (done.succeeded()) {
+                start.complete();
+            } else {
+                start.fail(done.cause());
+            }
+        });
+    }
+}

src/main/java/com/codingchili/Launcher.java

@@ -1,4 +1,4 @@
-package Model;
+package com.codingchili.Model;
 
 import io.vertx.core.json.JsonObject;
 
@@ -8,12 +8,14 @@
 
 /**
  * @author Robin Duda
+ *
+ * Handles application configuration.
  */
 public class Configuration {
     public static final int ELASTIC_PORT = configuration().getInteger("elastic_port");
     public static final String ELASTIC_HOST = configuration().getString("elastic_host");
     public static final int WEB_PORT = configuration().getInteger("web_port");
-    public static final String BUS_TRANSACTIONS = "bus.transactions";
+    public static final String INDEXING_ELASTICSEARCH = "bus.transactions";
     private static JsonObject configuration;
 
     private static JsonObject configuration() {

src/main/java/Model/Configuration.java renamed to src/main/java/com/codingchili/Model/Configuration.java

@@ -1,4 +1,6 @@
-package Model;
+package com.codingchili.Model;
+
+import java.util.logging.Logger;
 
 import io.vertx.core.AbstractVerticle;
 import io.vertx.core.Context;
@@ -9,8 +11,14 @@
 
 /**
  * @author Robin Duda
+ *
+ * Writes json data to elasticsearch for indexing.
  */
 public class ElasticWriter extends AbstractVerticle {
+    private static final String INDEX = "index";
+    private static final String BULK = "/_bulk";
+    private static final String ITEMS = "items";
+    private Logger logger = Logger.getLogger(getClass().getName());
     private Vertx vertx;
 
     @Override
@@ -20,21 +28,27 @@ public void init(Vertx vertx, Context context) {
 
     @Override
     public void start(Future<Void> start) {
-        processTransactions();
-
+        startSubmitListener();
+        logger.info("Started elastic writer");
         start.complete();
     }
 
-    private void processTransactions() {
-        vertx.eventBus().consumer(Configuration.BUS_TRANSACTIONS, handler -> {
+    private void startSubmitListener() {
+        vertx.eventBus().consumer(Configuration.INDEXING_ELASTICSEARCH, handler -> {
+            logger.info("Received file object, starting import..");
             JsonObject data = (JsonObject) handler.body();
-            String index = data.getString("index");
+            String index = data.getString(INDEX);
 
             vertx.createHttpClient().post(
-                    Configuration.ELASTIC_PORT, Configuration.ELASTIC_HOST, index + "/_bulk")
-                    .handler(response -> response.bodyHandler(body -> {
-                    }))
-                    .end(bulkQuery(data.getJsonArray("items"), index));
+                    Configuration.ELASTIC_PORT, Configuration.ELASTIC_HOST, index + BULK)
+                    .handler(response -> {
+                        logger.info(String.format("Submitted with result [%d] %s",
+                                response.statusCode(),
+                                response.statusMessage()));
+
+                        response.bodyHandler(body -> {});
+                    })
+                    .end(bulkQuery(data.getJsonArray(ITEMS), index));
         });
     }
 

src/main/java/Model/ElasticWriter.java renamed to src/main/java/com/codingchili/Model/ElasticWriter.java

@@ -1,4 +1,4 @@
-package Model;
+package com.codingchili.Model;
 
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
@@ -13,11 +13,16 @@
 
 /**
  * @author Robin Duda
+ *
+ * Parses xlsx files into json objects.
  */
 public class FileParser {
+    public static final String INDEX = "index";
+    public static final String ITEMS = "items";
     private JsonArray list = new JsonArray();
     private int columns;
     private int items;
+    private int offset;
 
     /**
      * Parses the contents of an XLSX into JSON.
@@ -31,6 +36,7 @@ public FileParser(byte[] bytes, int offset) throws ParserException {
             XSSFWorkbook workbook = new XSSFWorkbook(new ByteArrayInputStream(bytes));
             XSSFSheet sheet = workbook.getSheetAt(0);
 
+            this.offset = offset;
             this.columns = getColumnCount(sheet.getRow(offset));
             this.items = getItemCount(sheet, offset);
 
@@ -40,8 +46,8 @@ public FileParser(byte[] bytes, int offset) throws ParserException {
         }
     }
 
-    public JsonArray toJsonArray() {
-        return list;
+    public JsonObject toImportableObject() {
+        return new JsonObject().put(ITEMS, list).put(INDEX, offset);
     }
 
     private void readRows(XSSFSheet sheet, int columnRow) {

src/main/java/Model/FileParser.java renamed to src/main/java/com/codingchili/Model/FileParser.java

@@ -1,7 +1,9 @@
-package Model;
+package com.codingchili.Model;
 
 /**
  * @author Robin Duda
+ *
+ * Parser exception to be thrown when an error during xlsx parsing occurs.
  */
 public class ParserException extends Throwable {
 

src/main/java/Model/ParserException.java renamed to src/main/java/com/codingchili/Model/ParserException.java

@@ -1,3 +1,3 @@
 Manifest-Version: 1.0
-Main-Class: io.vertx.core.Launcher
+Main-Class: com.codingchili.Launcher