Adding support for netty/transport

core/add_project.sh

@@ -1,4 +1,16 @@
 #!/bin/bash
+function setup_netty() {
+    [ ! -d "app/ctest-netty-transport" ] && git clone https://github.com/HongxuMeng/netty.git app/ctest-netty-transport
+    cd app/ctest-netty-transport
+    git fetch && git checkout ctest-injection
+}
+
+function setup_netty_transport() {
+    setup_netty
+    home_dir=$PWD
+    cd $home_dir/transport
+    mvn clean package -DskipTests
+}
 
 function setup_hadoop() {
     [ ! -d "app/ctest-hadoop" ] && git clone https://github.com/xlab-uiuc/hadoop.git app/ctest-hadoop
@@ -64,7 +76,9 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            netty) setup_netty ;;
+            netty-transport) setup_netty_transport ;;
+            *) echo "Unexpected project: $project - only support netty, hadoop, hbase, zookeeper and alluxio." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+NETTY_TRANSPORT = "netty-transport"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_NETTY_TRANSPORT_DIR = os.path.join(APP_DIR, "ctest-netty-transport")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    NETTY_TRANSPORT: CTEST_NETTY_TRANSPORT_DIR
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    NETTY_TRANSPORT: "transport"    
 }
 
 
@@ -58,6 +62,9 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    NETTY_TRANSPORT: [
+        os.path.join(CTEST_NETTY_TRANSPORT_DIR, MODULE_SUBDIR[NETTY_TRANSPORT], SUREFIRE_SUBDIR)
+    ]
 }
 
 # default or deprecate conf path
@@ -74,7 +81,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    NETTY_TRANSPORT: os.path.join(DEFAULT_CONF_DIR, NETTY_TRANSPORT + "-default.tsv")
 }
 
 
@@ -96,6 +104,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    NETTY_TRANSPORT: [
+        os.path.join(CTEST_NETTY_TRANSPORT_DIR, "transport/ctest.json")
     ]
 }
 

core/default_configs/netty-transport-default.tsv

@@ -0,0 +1,13 @@
+connectTimeoutMillis	30000	
+maxMessagesPerRead	1	
+maxMessagesPerWrite	2147483647	
+writeSpinCount	16	
+allocator	io.netty.buffer.PooledByteBufAllocator	
+recvByteBufAllocator	io.netty.channel.AdaptiveRecvByteBufAllocator	
+autoRead	1	
+autoClose	true	
+writeBufferHighWaterMark	65536	
+writeBufferLowWaterMark	32768	
+writeBufferWaterMark	io.netty.channel.WriteBufferWaterMark	Obj param must be (int, int)
+messageSizeEstimator	io.netty.channel.DefaultMessageSizeEstimator	Obj param must be (int)
+pinEventExecutorPerGroup	true	

core/generate_ctest/inject.py

@@ -1,6 +1,7 @@
 """inject parameter, values into sw config"""
 
 import sys
+import json
 import xml.etree.ElementTree as ET
 
 sys.path.append("..")
@@ -14,7 +15,14 @@ def inject_config(param_value_pairs):
     for p, v in param_value_pairs.items():
         print(">>>>[ctest_core] injecting {} with value {}".format(p, v))
 
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [NETTY_TRANSPORT]:
+        for inject_path in INJECTION_PATH[project]:
+            print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
+            file = open(inject_path, "w")
+            json_object = json.dumps(param_value_pairs, indent=4)
+            file.write(json_object)
+            file.close()
+    elif project in [ZOOKEEPER, ALLUXIO]:
         for inject_path in INJECTION_PATH[project]:
             print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
             file = open(inject_path, "w")
@@ -41,7 +49,13 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO]:
+
+    if project in [NETTY_TRANSPORT]:
+        for inject_path in INJECTION_PATH[project]:
+            file = open(inject_path, "w")
+            file.write("{}\n")
+            file.close()
+    elif project in [ZOOKEEPER, ALLUXIO]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "netty-transport", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-netty-transport.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "../generate_value/netty-transport-generated-values.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_value/netty-transport-generated-values.tsv

@@ -0,0 +1,13 @@
+connectTimeoutMillis	15000	60000
+maxMessagesPerRead	0	2
+maxMessagesPerWrite	1073741823	4294967294
+writeSpinCount	8	32
+allocator	io.netty.buffer.UnpooledByteBufAllocator	SKIP
+recvByteBufAllocator	io.netty.channel.ServerChannelRecvByteBufAllocator	SKIP
+autoRead	1	SKIP
+autoClose	false	SKIP
+writeBufferHighWaterMark	32768	131072
+writeBufferLowWaterMark	16384	65536
+writeBufferWaterMark	SKIP	SKIP
+messageSizeEstimator	io.netty.channel.DefaultMessageSizeEstimator	SKIP
+pinEventExecutorPerGroup	false	SKIP

core/generate_value/value_generation.py

@@ -25,6 +25,9 @@ def read_tsv(module):
     if module == "zookeeper-server":
         assert len(params) == 32
         return 32
+    elif module == "netty-transport":
+        assert len(params) == 13
+        return 13
     else:
         assert len(params) == 90
         return 90
@@ -105,6 +108,8 @@ def print_params(module):
     f = open(module + output, "w")
     if module == "zookeeper-server":
         assert len(params) == 32
+    elif module == "netty-transport":
+        assert len(params) == 13
     else:
         assert len(params) >= 90
     for param in params:

core/identify_param/add_project.sh

@@ -1,4 +1,16 @@
 #!/bin/bash
+function setup_netty() {
+    [ ! -d "app/ctest-netty" ] && git clone https://github.com/HongxuMeng/netty.git app/ctest-netty
+    cd app/ctest-netty
+    git fetch && git checkout ctest-logging
+}
+
+function setup_netty_transport() {
+    setup_netty
+    home_dir=$PWD
+    cd $home_dir/transport
+    mvn clean package -DskipTests
+}
 
 function setup_hadoop() {
     [ ! -d "app/ctest-hadoop" ] && git clone https://github.com/xlab-uiuc/hadoop.git app/ctest-hadoop
@@ -63,7 +75,9 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            netty) setup_netty ;;
+            netty_transport) setup_netty_transport ;;
+            *) echo "Unexpected project: $project - only support netty, hadoop, hbase, zookeeper and alluxio." ;;
         esac
     fi
 }

core/identify_param/collector.py

@@ -31,7 +31,11 @@ def parse_getter_record_file(self):
 
             if param not in self.param_getter_map.keys():
                 self.param_getter_map[param] = set()
-            self.param_getter_map[param].add(class_pound_method)        
+            self.param_getter_map[param].add(class_pound_method)
+
+            # if class_pound_method not in self.param_getter_map.keys():
+            #     self.param_getter_map[class_pound_method] = set()
+            # self.param_getter_map[class_pound_method].add(param)
 
     def parse_setter_record_file(self):
         for line in open(self.setter_record_file).readlines():
@@ -43,6 +47,10 @@ def parse_setter_record_file(self):
             if param not in self.param_setter_map.keys():
                 self.param_setter_map[param] = set()
             self.param_setter_map[param].add(class_pound_method)
+
+            # if class_pound_method not in self.param_setter_map.keys():
+            #     self.param_setter_map[class_pound_method] = set()
+            # self.param_setter_map[class_pound_method].add(param)
 
     def generate_unset_getter_mapping(self):
         for key in self.param_getter_map.keys():
@@ -70,6 +78,8 @@ def generate_mapping(self):
     def sanity_check(self):
         for key in self.param_unset_getter_map.keys():
             assert key in self.params, "error"
+            # for value in self.param_unset_getter_map[key]:
+            #     assert value in self.params, "error"
             if key not in self.param_setter_map.keys():
                 assert self.param_unset_getter_map[key] == self.param_getter_map[key]
             else:

core/identify_param/constant.py

@@ -7,20 +7,23 @@
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZOOKEEPER_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_NETTY_DIR = os.path.join(APP_DIR, "ctest-netty")
 
 MODULE_PATH = {
     "hadoop-common": CTEST_HADOOP_DIR,
     "hadoop-hdfs": CTEST_HADOOP_DIR,
     "hbase-server": CTEST_HBASE_DIR,
-    "alluxio-core": CTEST_ALLUXIO_DIR
+    "alluxio-core": CTEST_ALLUXIO_DIR,
+    "netty-transport": CTEST_NETTY_DIR
 }
 
 SRC_SUBDIR = {
     "hadoop-common": "hadoop-common-project/hadoop-common",
     "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs",
     "hbase-server": "hbase-server",
     "zookeeper-server": "zookeeper-server",
-    "alluxio-core": "core"
+    "alluxio-core": "core",
+    "netty-transport": "transport"
 }
 
 MVN_TEST_PATH = {
@@ -29,14 +32,16 @@
     "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]),
     "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]),
     "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]),
+    "netty-transport": os.path.join(CTEST_NETTY_DIR, SRC_SUBDIR["netty-transport"])
 }
 
 LOCAL_CONF_PATH = {
     "hadoop-common": "results/hadoop-common/conf_params.txt",
     "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt",
     "hbase-server": "results/hbase-server/conf_params.txt",
     "zookeeper-server": "results/zookeeper-server/conf_params.txt",
-    "alluxio-core": "results/alluxio-core/conf_params.txt"
+    "alluxio-core": "results/alluxio-core/conf_params.txt",
+    "netty-transport": "results/netty-transport/conf_params.txt"
 }
 
 SUREFIRE_SUBDIR = "target/surefire-reports/*"
@@ -63,6 +68,9 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR)
+    ],
+    "netty-transport": [
+        os.path.join(CTEST_NETTY_DIR, SRC_SUBDIR["netty-transport"], SUREFIRE_SUBDIR)
     ]
 }
 
@@ -83,5 +91,8 @@
     ],
     "alluxio-core": [
         os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX)
+    ],
+    "netty-transport": [
+        os.path.join("surefire-reports/common/netty-transport", LOCAL_SUREFIRE_SUFFIX)
     ]
 }

core/identify_param/identify_param.sh

@@ -12,9 +12,9 @@ function main() {
       usage
     else
         case $project in
-            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;;
+            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | netty-transport) python3 runner.py $project; python3 collector.py $project ;;
             -h | --help) usage ;;
-            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;;
+            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core, and netty-transport" ;;
         esac
     fi
 }

core/identify_param/results/netty-transport/conf_params.txt

@@ -0,0 +1,13 @@
+connectTimeoutMillis
+maxMessagesPerRead
+maxMessagesPerWrite
+writeSpinCount
+allocator
+recvByteBufAllocator
+autoRead
+autoClose
+writeBufferHighWaterMark
+writeBufferLowWaterMark
+writeBufferWaterMark
+messageSizeEstimator
+pinEventExecutorPerGroup