From 2ead364c5da01032cfd9f8aa97c10b88e3dc2519 Mon Sep 17 00:00:00 2001
From: emk <nh43499@hertz.net>
Date: Wed, 28 Aug 2019 13:47:49 +0100
Subject: [PATCH 1/2] updates for cluster autoscaler functionality. fixed
 issues with index.js and common.js. updated lambda function to nodejs8.10.
 included deployment of kubernetes/autoscaler deployment. created configmap
 for install and delete scripts to remove from javascript.

---
 autoscaling.tf                       |  91 ++++++++++++++++++++-
 autoscaling/autoscalinggroup.tf      |  53 ++++++------
 autoscaling/cloudwatch.tf            |   4 +-
 autoscaling/lambda.tf                |   2 +-
 autoscaling/lambda/common.js         |  65 ++++++++++-----
 autoscaling/lambda/index.js          |  89 +++++++++++---------
 autoscaling/lambda/job-tmpl.yaml     |  21 +++++
 autoscaling/variables.tf             |  21 +++++
 efs.tf                               |   4 +-
 icp-deploy.tf                        |  22 ++++-
 instances.tf                         |   6 +-
 scripts/asg-configmap.yaml           |  47 +++++++++++
 scripts/cluster-autoscaler-rbac.yaml | 118 +++++++++++++++++++++++++++
 scripts/create_client_cert.sh        |  41 +++++++++-
 scripts/start_install.sh             |  13 ++-
 15 files changed, 493 insertions(+), 104 deletions(-)
 create mode 100644 scripts/asg-configmap.yaml
 create mode 100644 scripts/cluster-autoscaler-rbac.yaml

diff --git a/autoscaling.tf b/autoscaling.tf
index 6b2410c..9d0b89a 100644
--- a/autoscaling.tf
+++ b/autoscaling.tf
@@ -24,14 +24,17 @@ module "icpautoscaling" {
     #icpuser         = "aws_lb_target_group_attachment.master-8001.arn" // attempt at workaround for missing depends on
 
     kube_api_url    = "https://${aws_lb.icp-console.dns_name}:8001"
+    docker_registry = "${var.user_provided_cert_dns != "" ? var.user_provided_cert_dns : aws_lb.icp-console.dns_name}:8500"
 
     aws_region            = "${var.aws_region}"
     azs                   = ["${var.azs}"]
-    ami                   = "${var.worker["ami"] != "" ? var.worker["ami"] : lookup(local.default_searched_ami, var.ami, var.ami)}"
+    ami                   = "${var.worker["ami"] != "" ? var.worker["ami"] : local.default_ami }"
     worker_root_disk_size = "${var.worker["disk"]}"
     worker_docker_vol_size = "${var.worker["docker_vol"]}"
     key_name              = "${var.key_name}"
     instance_type         = "${var.worker["type"]}"
+    ebs_optimized         = "${var.worker["ebs_optimized"]}"
+    instance_name         = "${var.instance_name}"
     security_groups = [
       "${aws_security_group.default.id}"
     ]
@@ -44,4 +47,90 @@ module "icpautoscaling" {
     image_location            = "${local.image_package_uri}"
     icp_inception_image       = "${var.icp_inception_image}"
     lambda_s3_bucket          = "${local.lambda_s3_bucket}"
+    icp_config_s3_bucket      = "${aws_s3_bucket.icp_config_backup.id}"
+    asg_tags                  = ["${data.null_data_source.asg-tags.*.outputs}"]
 }
+
+data "null_data_source" "asg-tags" {
+  count = "${length(keys(var.default_tags))}"
+  inputs = {
+    key                 = "${element(keys(var.default_tags), count.index)}"
+    value               = "${element(values(var.default_tags), count.index)}"
+    propagate_at_launch = "true"
+  }
+}
+
+resource "aws_s3_bucket_object" "icp_cluster_autoscaler_yaml" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "scripts/cluster-autoscaler-deployment.yaml"
+  content = <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    app: cluster-autoscaler
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  template:
+    metadata:
+      labels:
+        app: cluster-autoscaler
+    spec:
+      serviceAccountName: cluster-autoscaler
+      containers:
+        - image: k8s.gcr.io/cluster-autoscaler:v1.2.2
+          name: cluster-autoscaler
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi
+          command:
+            - ./cluster-autoscaler
+            - --v=4
+            - --stderrthreshold=info
+            - --cloud-provider=aws
+            - --skip-nodes-with-local-storage=false
+            - --skip-nodes-with-system-pods=false
+            - --expander=least-waste
+            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,kubernetes.io/cluster/${random_id.clusterid.hex}
+            - --balance-similar-node-groups=true
+          volumeMounts:
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs/ca-certificates.crt
+              readOnly: true
+          imagePullPolicy: "Always"
+      nodeSelector:
+        master: "true"
+      tolerations:
+      - effect: NoSchedule
+        key: dedicated
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs/ca-bundle.crt"
+EOF
+}
+
+resource "aws_s3_bucket_object" "asg_configmap" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "scripts/asg-configmap.yaml"
+  source = "${path.module}/scripts/asg-configmap.yaml"
+}
+
+resource "aws_s3_bucket_object" "cluster_autoscaler_rbac_yaml" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "scripts/cluster-autoscaler-rbac.yaml"
+  source = "${path.module}/scripts/cluster-autoscaler-rbac.yaml"
+}
+
diff --git a/autoscaling/autoscalinggroup.tf b/autoscaling/autoscalinggroup.tf
index 0729741..e61d754 100644
--- a/autoscaling/autoscalinggroup.tf
+++ b/autoscaling/autoscalinggroup.tf
@@ -10,7 +10,7 @@ resource "aws_launch_configuration" "icp_worker_lc" {
 
   security_groups = ["${var.security_groups}"]
 
-  ebs_optimized = true
+  ebs_optimized = "${var.ebs_optimized}"
   root_block_device {
     volume_size = "${var.worker_root_disk_size}"
   }
@@ -27,15 +27,17 @@ resource "aws_launch_configuration" "icp_worker_lc" {
 packages:
   - unzip
   - python
+  - bind-utils
 rh_subscription:
   enable-repo: rhui-REGION-rhel-server-optional
 write_files:
-  - path: /tmp/bootstrap.sh
+  - path: /tmp/bootstrap-node.sh
     permissions: '0755'
     encoding: b64
-    content: ${base64encode(file("${path.module}/../scripts/bootstrap.sh"))}
+    content: ${base64encode(file("${path.module}/../scripts/bootstrap-node.sh"))}
 runcmd:
-  - /tmp/bootstrap.sh ${var.docker_package_location != "" ? "-p ${var.docker_package_location}" : "" } -d /dev/xvdx ${var.image_location != "" ? "-i ${var.image_location}" : "" } -s ${var.icp_inception_image}
+  - /tmp/bootstrap-node.sh -c ${var.icp_config_s3_bucket} -s "bootstrap.sh"
+  - /tmp/icp_scripts/bootstrap.sh ${var.docker_package_location != "" ? "-p ${var.docker_package_location}" : "" } -d /dev/xvdx ${var.image_location != "" ? "-i ${var.image_location}" : "" } -s ${var.icp_inception_image}
 users:
   - default
   - name: icpdeploy
@@ -54,57 +56,58 @@ EOF
 }
 
 resource "aws_autoscaling_group" "icp_worker_asg" {
-  count = "${var.enabled ? 1 : 0}"
-  name                 = "icp-worker-asg-${var.cluster_id}"
+  count = "${length(var.azs)}"
+  name                 = "icp-worker-asg-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
   launch_configuration = "${aws_launch_configuration.icp_worker_lc.name}"
   min_size             = 0
   max_size             = 20
   force_delete         = true
 
-  availability_zones   = "${formatlist("%v%v", var.aws_region, var.azs)}"
-  vpc_zone_identifier  = ["${var.private_subnet_ids}"]
+  vpc_zone_identifier  = ["${element(var.private_subnet_ids, count.index)}"]
 
-  tags = [
-    {
-      key                 = "kubernetes.io/cluster/${var.cluster_id}",
-      value               = "${var.cluster_id}",
-      propagate_at_launch = true
-    }
-  ]
+  tags = ["${concat(
+    var.asg_tags,
+    list(map("key", "k8s.io/cluster-autoscaler/enabled", "value", "${var.enabled}", "propagate_at_launch", "false")),
+    list(map("key", "kubernetes.io/cluster/${var.cluster_id}", "value", "${var.cluster_id}", "propagate_at_launch", "true"))
+  )}"]
 }
 
 resource "aws_autoscaling_lifecycle_hook" "icp_add_worker_hook" {
-  count = "${var.enabled ? 1 : 0}"
-  name                   = "icp-workernode-added-${var.cluster_id}"
-  autoscaling_group_name = "${aws_autoscaling_group.icp_worker_asg.name}"
+  count = "${length(var.azs)}"
+  name                   = "icp-workernode-added-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
+  autoscaling_group_name = "${element(aws_autoscaling_group.icp_worker_asg.*.name, count.index)}"
   default_result         = "ABANDON"
   heartbeat_timeout      = 3600
   lifecycle_transition   = "autoscaling:EC2_INSTANCE_LAUNCHING"
 
   notification_metadata = <<EOF
 {
-  "icp_inception_image": "${var.icp_inception_image}",
+  "icp_inception_image": "${var.docker_registry}/${var.icp_inception_image}",
   "docker_package_location": "${var.docker_package_location}",
   "image_location": "${var.image_location}",
-  "cluster_backup": "icpbackup-${var.cluster_id}"
+  "cluster_backup": "icpbackup-${var.cluster_id}",
+  "cluster_id": "${var.cluster_id}",
+  "instance_name": "${var.instance_name}"
 }
 EOF
 }
 
 resource "aws_autoscaling_lifecycle_hook" "icp_del_worker_hook" {
-  count = "${var.enabled ? 1 : 0}"
-  name                   = "icp-workernode-removed-${var.cluster_id}"
-  autoscaling_group_name = "${aws_autoscaling_group.icp_worker_asg.name}"
+  count = "${length(var.azs)}"
+  name                   = "icp-workernode-removed-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
+  autoscaling_group_name = "${element(aws_autoscaling_group.icp_worker_asg.*.name, count.index)}"
   default_result         = "ABANDON"
   heartbeat_timeout      = 3600
   lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
 
   notification_metadata = <<EOF
 {
-  "icp_inception_image": "${var.icp_inception_image}",
+  "icp_inception_image": "${var.docker_registry}/${var.icp_inception_image}",
   "docker_package_location": "${var.docker_package_location}",
   "image_location": "${var.image_location}",
-  "cluster_backup": "icpbackup-${var.cluster_id}"
+  "cluster_backup": "icpbackup-${var.cluster_id}",
+  "cluster_id": "${var.cluster_id}",
+  "instance_name": "${var.instance_name}"
 }
 EOF
 }
diff --git a/autoscaling/cloudwatch.tf b/autoscaling/cloudwatch.tf
index 97056c4..b9ac283 100644
--- a/autoscaling/cloudwatch.tf
+++ b/autoscaling/cloudwatch.tf
@@ -13,7 +13,7 @@ resource "aws_cloudwatch_event_rule" "icp_worker_node_added_event" {
   ],
   "detail": {
     "AutoScalingGroupName": [
-      "${aws_autoscaling_group.icp_worker_asg.name}"
+      ${join(",", formatlist("\"%v\"", aws_autoscaling_group.icp_worker_asg.*.name))}
     ]
   }
 }
@@ -35,7 +35,7 @@ resource "aws_cloudwatch_event_rule" "icp_worker_node_remove_event" {
   ],
   "detail": {
     "AutoScalingGroupName": [
-      "${aws_autoscaling_group.icp_worker_asg.name}"
+      ${join(",", formatlist("\"%v\"", aws_autoscaling_group.icp_worker_asg.*.name))}
     ]
   }
 }
diff --git a/autoscaling/lambda.tf b/autoscaling/lambda.tf
index 8836f86..d8236e3 100644
--- a/autoscaling/lambda.tf
+++ b/autoscaling/lambda.tf
@@ -47,7 +47,7 @@ resource "aws_lambda_function" "icp_autoscale" {
   function_name    = "icp-worker-autoscale-${var.cluster_id}"
   role             = "${local.iam_lambda_role_arn}"
   handler          = "index.handler"
-  runtime          = "nodejs6.10"
+  runtime          = "nodejs8.10"
   timeout          = 10
 
   vpc_config {
diff --git a/autoscaling/lambda/common.js b/autoscaling/lambda/common.js
index 4c84138..9744151 100644
--- a/autoscaling/lambda/common.js
+++ b/autoscaling/lambda/common.js
@@ -1,29 +1,42 @@
 var aws = require('aws-sdk');
 var kubeapi = require('kubernetes-client');
 var fs = require('fs');
-
+var region; 
 aws.config.update({region: region});
 aws.config.setPromisesDependency(Promise);
 
 function get_instance_ip(region, instance_id) {
   var ec2 = new aws.EC2({apiVersion: '2016-11-15'});
 
-  console.log("InstanceID: " + instance_id);
+  console.log("get_instance_ip: Region: " + region);
+  console.log("get_instance_ip: InstanceID: " + instance_id);
 
   var params = {
-    DryRun: false,
-    InstanceIds: [ instance_id ],
+    //DryRun: false,
+    InstanceIds: [ instance_id ]
   };
 
-  return ec2.describeInstances(params, function(err, result) {
-    if (err) {
-      console.log(err, err.stack);
+  var request = ec2.describeInstances(params);
+  
+  var promise = request.promise();
+  
+  // handle promise's fulfilled/rejected states
+  promise.then(
+    function(data) {
+      console.log("get_instance_ip: Instance IP address is: " + data.Reservations[0].Instances[0].PrivateIpAddress);
+      console.log("get_instance_ip: " + JSON.stringify(data, null, 2));
+      
+      //return data.Reservations[0].Instances[0].PrivateIpAddress;
+      /* process the data */
+    },
+    function(err) {
+      /* handle the error */
+      console.log("get_instance_ip: " + err, err.stack);
       throw err;
-    } else {
-      console.log("Instance IP address is: " + result.Reservations[0].Instances[0].PrivateIpAddress);
-      return result.Reservations[0].Instances[0].PrivateIpAddress;
     }
-  });
+  );
+  
+  return promise;
 }
 
 function get_bucket_object(bucketName, key) {
@@ -34,16 +47,26 @@ function get_bucket_object(bucketName, key) {
     Key: key
   };
 
-  s3.getObject(params, function(err, data) {
-    if (err) {
-      console.log(err, err.stack);
+  var request = s3.getObject(params);
+  
+  var promise = request.promise();
+  
+  // handle promise's fulfilled/rejected states
+  promise.then(
+    function(data) {
+      console.log("get_bucket_object: body is: " + data.Body);
+      
+      //return data.Reservations[0].Instances[0].PrivateIpAddress;
+      /* process the data */
+    },
+    function(err) {
+      /* handle the error */
+      console.log("get_bucket_object: " + err, err.stack);
       throw err;
-    } else {
-      console.log(data);           // successful response
-
-      return data.Body;
     }
-  });
+  );
+  
+  return promise;
 }
 
 function create_job(ca_crt, client_cert, client_key, job) {
@@ -77,8 +100,10 @@ function fail_autoscaling(params) {
     .then(function(result) {
         console.log("competed lifecycle action");
     });
-};
+}
 
 module.exports.get_instance_ip = get_instance_ip;
 module.exports.create_job = create_job;
 module.exports.fail_autoscaling = fail_autoscaling;
+module.exports.get_bucket_object = get_bucket_object;
+
diff --git a/autoscaling/lambda/index.js b/autoscaling/lambda/index.js
index a06ad75..88bd23c 100644
--- a/autoscaling/lambda/index.js
+++ b/autoscaling/lambda/index.js
@@ -3,7 +3,7 @@ console.log('Loading function');
 var uuid = require('uuid');
 var fs = require('fs');
 var yaml = require('js-yaml');
-var common = require('./common.js')
+var common = require('./common.js');
 
 var job_tmpl = yaml.safeLoad(fs.readFileSync('./job-tmpl.yaml', 'utf8'));
 
@@ -40,7 +40,7 @@ exports.handler = (event, context, callback) => {
         /* not interested in this event */
         return;
     }
-
+    
     var promises = [];
 
     promises.push(common.get_instance_ip(event.region, instanceId));
@@ -48,16 +48,17 @@ exports.handler = (event, context, callback) => {
     promises.push(common.get_bucket_object(process.env.s3_bucket, "lambda-cert.pem"));
     promises.push(common.get_bucket_object(process.env.s3_bucket, "lambda-key.pem"));
 
-    return Promises.all(promises)
+    return Promise.all(promises)
     .then(function(result) {
       /* try to create a batch job in kube */
       if (event.detail.LifecycleTransition === "autoscaling:EC2_INSTANCE_TERMINATING") {
-        console.log("scaling down node " + result[0]);
+        console.log("scaling down node " + result[0].Reservations[0].Instances[0].PrivateIpAddress);
         return create_delete_node_job(result, event);
       }
 
       if (event.detail.LifecycleTransition === "autoscaling:EC2_INSTANCE_LAUNCHING") {
-        console.log("scaling up node " + result[0]);
+        console.log("scaling up cluster using node " + result[0].Reservations[0].Instances[0].PrivateIpAddress);
+
         return create_add_node_job(result, event);
       }
     }).catch(function(err) {
@@ -70,10 +71,13 @@ exports.handler = (event, context, callback) => {
 };
 
 function create_add_node_job(params, event) {
-  var privateIp = params[0];
-  var jobName = 'add-node-' + privateIp.replace(new RegExp(/\./, 'g'), "-") + "-" + uuid.v4().substring(0, 7);
+  var privateIp = params[0].Reservations[0].Instances[0].PrivateIpAddress;
+  //var jobName = 'add-node-' + privateIp.replace(new RegExp(/\./, 'g'), "-") + "-" + uuid.v4().substring(0, 7);
   var metadataStr = unescape(event.detail.NotificationMetadata);
   var metadata = JSON.parse(metadataStr);
+  
+  var instance_name = metadata.instance_name + "-" + metadata.cluster_id + "-worker-" + event.detail.EC2InstanceId.replace("i-", "");
+  var jobName = 'add-node-' + instance_name + "-" + uuid.v4().substring(0, 7);
 
   job_tmpl.metadata.name = jobName;
   job_tmpl.metadata.labels.run = jobName;
@@ -81,7 +85,7 @@ function create_add_node_job(params, event) {
 
   // use installer image
   job_tmpl.spec.template.spec.containers[0].image = metadata.icp_inception_image;
-  job_tmpl.spec.template.spec.containers[0].command = [ "/bin/bash", "-c" ];
+  job_tmpl.spec.template.spec.containers[0].command = [ "/bin/bash", "-c", "/installer/cluster/add_worker.sh" ];
   job_tmpl.spec.template.spec.containers[0].env = [
     {
       name: "LICENSE",
@@ -119,6 +123,10 @@ function create_add_node_job(params, event) {
       name: "INSTANCEID",
       value: event.detail.EC2InstanceId
     },
+    {
+      name: "INSTANCE_NAME",
+      value: instance_name
+    },
     {
       name: "REGION",
       value: event.region
@@ -128,30 +136,34 @@ function create_add_node_job(params, event) {
       value: "false"
     }
   ];
-
-  job_tmpl.spec.template.spec.containers[0].args = [
-    "curl https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -o /tmp/awscli-bundle.zip;" +
-    "unzip /tmp/awscli-bundle.zip -d /tmp; " +
-    "/tmp/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws; " +
-    "/usr/local/bin/aws s3 cp --recursive s3://${CLUSTER_BACKUP} /installer/cluster; " +
-    "rm -f /installer/cluster/.install.lock; " +
-    "chmod 400 /installer/cluster/ssh_key; " +
-    "ansible -i /opt/ibm/cluster/hosts ${NODE_IP} --private-key /opt/ibm/cluster/ssh_key -u icpdeploy -b -m wait_for -a 'path=/var/lib/cloud/instance/boot-finished timeout=18000; " +
-    "crudini --set /installer/cluster/hosts worker ${NODE_IP}; " +
-    "/installer/installer.sh install -l ${NODE_IP} && " +
-    "/usr/local/bin/aws --region ${REGION} autoscaling complete-lifecycle-action --lifecycle-hook-name ${LIFECYCLEHOOKNAME} --lifecycle-action-token ${LIFECYCLEACTIONTOKEN} --auto-scaling-group-name ${ASGNAME} --lifecycle-action-result CONTINUE --instance-id ${INSTANCEID} && " +
-    "/usr/local/bin/aws s3 sync /installer/cluster s3://${CLUSTER_BACKUP}"
+  
+  job_tmpl.spec.template.spec.containers[0].volumeMounts = [
+    {
+      mountPath: "/installer/cluster/add_worker.sh",
+      name: "autoscaler-config",
+      subPath: "add_worker.sh"
+    }
   ];
+  
+  job_tmpl.spec.template.spec.volumes[0].configMap.defaultMode = 493;
+  job_tmpl.spec.template.spec.volumes[0].configMap.items[0].key = "add_worker.sh";
+  job_tmpl.spec.template.spec.volumes[0].configMap.items[0].path = "add_worker.sh";
+  job_tmpl.spec.template.spec.volumes[0].configMap.name = "autoscaler-config";
+  job_tmpl.spec.template.spec.volumes[0].name = "autoscaler-config";
 
   console.log("Sending job: " + JSON.stringify(job_tmpl, 2));
-  return common.create_job(params[1], params[2], params[3], job_tmpl);
+  console.log("certificate is: " + params[1].Body);
+  return common.create_job(params[1].Body, params[2].Body, params[3].Body, job_tmpl);
 }
 
 function create_delete_node_job(params, event) {
-  var privateIp = params[0];
-  var jobName = 'delete-node-' + privateIp.replace(new RegExp(/\./, 'g'), "-") + "-" + uuid.v4().substring(0, 7);
+  var privateIp = params[0].Reservations[0].Instances[0].PrivateIpAddress;
+  //var jobName = 'delete-node-' + privateIp.replace(new RegExp(/\./, 'g'), "-") + "-" + uuid.v4().substring(0, 7);
   var metadataStr = unescape(event.detail.NotificationMetadata);
   var metadata = JSON.parse(metadataStr);
+  
+  var instance_name = metadata.instance_name + "-" + metadata.cluster_id + "-worker-" + event.detail.EC2InstanceId.replace("i-", "");
+  var jobName = 'delete-node-' + instance_name + "-" + uuid.v4().substring(0, 7);
 
   job_tmpl.metadata.name = jobName;
   job_tmpl.metadata.labels.run = jobName;
@@ -159,7 +171,7 @@ function create_delete_node_job(params, event) {
 
   // use installer image
   job_tmpl.spec.template.spec.containers[0].image = metadata.icp_inception_image;
-  job_tmpl.spec.template.spec.containers[0].command = [ "/bin/bash", "-c" ];
+  job_tmpl.spec.template.spec.containers[0].command = [ "/bin/bash", "-c", "/installer/cluster/remove_worker.sh" ];
   job_tmpl.spec.template.spec.containers[0].env = [
     {
       name: "LICENSE",
@@ -203,22 +215,22 @@ function create_delete_node_job(params, event) {
     }
   ];
 
-  job_tmpl.spec.template.spec.containers[0].args = [
-    "curl https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -o /tmp/awscli-bundle.zip;" +
-    "unzip /tmp/awscli-bundle.zip -d /tmp; " +
-    "/tmp/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws; " +
-    "aws s3 cp --recursive s3://${CLUSTER_BACKUP} /installer/cluster; " +
-    "chmod 400 /installer/cluster/ssh_key; " +
-    "crudini --set /installer/cluster/hosts worker ${NODE_IP}; " +
-    "rm -f /installer/cluster/.install.lock; " +
-    "/installer/installer.sh uninstall -l ${NODE_IP} && " +
-    "aws --region ${REGION} autoscaling complete-lifecycle-action --lifecycle-hook-name ${LIFECYCLEHOOKNAME} --lifecycle-action-token ${LIFECYCLEACTIONTOKEN} --auto-scaling-group-name ${ASGNAME} --lifecycle-action-result CONTINUE --instance-id ${INSTANCEID} && " +
-    "crudini --del /installer/cluster/hosts worker ${NODE_IP} && " +
-    "/usr/local/bin/aws s3 sync /installer/cluster s3://${CLUSTER_BACKUP}"
+  job_tmpl.spec.template.spec.containers[0].volumeMounts = [
+    {
+      mountPath: "/installer/cluster/remove_worker.sh",
+      name: "autoscaler-config",
+      subPath: "remove_worker.sh"
+    }
   ];
+  
+  job_tmpl.spec.template.spec.volumes[0].configMap.defaultMode = 493;
+  job_tmpl.spec.template.spec.volumes[0].configMap.items[0].key = "remove_worker.sh";
+  job_tmpl.spec.template.spec.volumes[0].configMap.items[0].path = "remove_worker.sh";
+  job_tmpl.spec.template.spec.volumes[0].configMap.name = "autoscaler-config";
+  job_tmpl.spec.template.spec.volumes[0].name = "autoscaler-config";
 
   console.log("Sending job: " + JSON.stringify(job_tmpl, 2));
-  return common.create_job(params[1], params[2], params[3], job_tmpl);
+  return common.create_job(params[1].Body, params[2].Body, params[3].Body, job_tmpl);
 }
 
 process.on('unhandledRejection', function(error) {
@@ -254,3 +266,4 @@ exports.handler(sample_event, null, function(err, result) {
   }
 });
 */
+
diff --git a/autoscaling/lambda/job-tmpl.yaml b/autoscaling/lambda/job-tmpl.yaml
index cdf640e..d438c6e 100644
--- a/autoscaling/lambda/job-tmpl.yaml
+++ b/autoscaling/lambda/job-tmpl.yaml
@@ -36,4 +36,25 @@ spec:
         env:
         - name: LICENSE
           value: accept
+        volumeMounts:
+        - mountPath: /installer/cluster/add_worker.sh
+          name: autoscaler-config
+          subPath: add_worker.sh
+      volumes:
+      - configMap:
+          defaultMode: 493
+          items:
+          - key: add_worker.sh
+            path: add_worker.sh
+          name: autoscaler-config
+        name: autoscaler-config
+      nodeSelector:
+        master: "true"
+      tolerations:
+      - effect: NoSchedule
+        key: dedicated
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists  
   backoffLimit: 4
+
diff --git a/autoscaling/variables.tf b/autoscaling/variables.tf
index ab64631..b71295b 100644
--- a/autoscaling/variables.tf
+++ b/autoscaling/variables.tf
@@ -25,6 +25,10 @@ variable "instance_type" {
   default = ""
 }
 
+variable "ebs_optimized" {
+  default = ""
+}
+
 variable "cluster_id" {
   default = ""
 }
@@ -84,10 +88,27 @@ variable "icp_inception_image" {
   default     = "ibmcom/icp-inception:2.1.0.2-ee"
 }
 
+variable "awscli" {
+  default = ""
+}
+
 variable "lambda_s3_bucket" {
   default = ""
 }
 
+variable "instance_name" {
+  default = ""
+}
+
+variable "icp_config_s3_bucket" {
+  default = ""
+}
+
+# Default tags to apply to resources
+variable "asg_tags" {
+  default = []
+}
+
 variable "enabled" {
   default = true
 }
diff --git a/efs.tf b/efs.tf
index dc65607..a727f40 100644
--- a/efs.tf
+++ b/efs.tf
@@ -3,7 +3,7 @@ resource "aws_efs_file_system" "icp-registry" {
   creation_token = "icp-${random_id.clusterid.hex}-registry"
   tags = "${merge(
     var.default_tags,
-    map("Name", "icp-registry")
+    map("Name", "${format("icp-${random_id.clusterid.hex}-registry")}")
   )}"
 }
 
@@ -19,7 +19,7 @@ resource "aws_efs_file_system" "icp-audit" {
   creation_token = "icp-${random_id.clusterid.hex}-audit"
   tags = "${merge(
     var.default_tags,
-    map("Name", "icp-audit")
+    map("Name", "${format("icp-${random_id.clusterid.hex}-audit")}")
   )}"
 }
 
diff --git a/icp-deploy.tf b/icp-deploy.tf
index 5e1b888..5bd39c1 100644
--- a/icp-deploy.tf
+++ b/icp-deploy.tf
@@ -124,7 +124,11 @@ icp_config_file = "./icp-terraform-config.yaml"
 # We will let terraform generate a new ssh keypair
 # for boot master to communicate with worker and proxy nodes
 # during ICP deployment
-generate_key = true
+generate_key = false
+icp_pub_key = "${chomp(tls_private_key.installkey.public_key_openssh)}"
+icp_priv_key = <<EOKEY
+${tls_private_key.installkey.private_key_pem}
+EOKEY
 
 # SSH user and key for terraform to connect to newly created VMs
 # ssh_key is the private key corresponding to the public assumed to be included in the template
@@ -132,9 +136,25 @@ ssh_user        = "icpdeploy"
 ssh_key_base64  = "${base64encode(tls_private_key.installkey.private_key_pem)}"
 ssh_agent       = false
 
+hooks = {
+  cluster-preconfig  = ["echo -n"]
+  cluster-postconfig = ["echo -n"]
+  boot-preconfig     = ["echo -n"]
+  preinstall         = ["echo -n"]
+  postinstall        = [
+    "sudo /tmp/icp_scripts/create_client_cert.sh -b ${local.lambda_s3_bucket} -i ${var.icp_inception_image} -k ${aws_network_interface.mastervip.0.private_ip}",
+    "/usr/local/bin/aws autoscaling set-desired-capacity --auto-scaling-group-name icp-worker-asg-${random_id.clusterid.hex} --region ${var.aws_region} --desired-capacity 1"
+  ]
+}
+
 EOF
 }
 
+resource "aws_s3_bucket_object" "ssh_key" {
+  bucket = "${aws_s3_bucket.icp_config_backup.id}"
+  key    = "ssh_key"
+  content = "${tls_private_key.installkey.private_key_pem}"
+}
 
 resource "tls_private_key" "installkey" {
   algorithm   = "RSA"
diff --git a/instances.tf b/instances.tf
index db168aa..4ffe0bd 100644
--- a/instances.tf
+++ b/instances.tf
@@ -19,7 +19,7 @@ locals  {
   docker_package_uri = "${substr(var.docker_package_location, 0, min(2, length(var.docker_package_location))) == "s3" ?
     var.docker_package_location :
       var.docker_package_location == "" ? "" : "s3://${element(concat(aws_s3_bucket.icp_binaries.*.id, list("")), 0)}/icp-docker.bin"}"
-  lambda_s3_bucket = "${element(concat(aws_s3_bucket.icp_lambda.*.id, list("")), 0)}"
+  lambda_s3_bucket = "${var.enable_autoscaling ? element(concat(aws_s3_bucket.icp_lambda.*.id, list("")), 0) : "" }"
   default_searched_ami = {
     "ubuntu"  = "${data.aws_ami.ubuntu.id}"
     "rhel"    = "${data.aws_ami.rhel.id}"
@@ -179,14 +179,14 @@ write_files:
   content: ${base64encode(file("${path.module}/scripts/bootstrap-node.sh"))}
 runcmd:
 - hostnamectl set-hostname $(curl http://169.254.169.254/2016-09-02/meta-data/local-hostname)
-- /tmp/bootstrap-node.sh -c ${aws_s3_bucket.icp_config_backup.id} -s "bootstrap.sh functions.sh ${count.index == 0 ? "start_install.sh" : ""} ${count.index == 0 && var.enable_autoscaling ? "create_client_cert.sh" : ""}"
+- /tmp/bootstrap-node.sh -c ${aws_s3_bucket.icp_config_backup.id} -s "bootstrap.sh functions.sh ${count.index == 0 ? "start_install.sh" : ""} ${count.index == 0 && var.enable_autoscaling ? "create_client_cert.sh asg-configmap.yaml cluster-autoscaler-rbac.yaml cluster-autoscaler-deployment.yaml" : ""}"
 - /tmp/icp_scripts/bootstrap.sh ${local.docker_package_uri != "" ? "-p ${local.docker_package_uri}" : "" } -d /dev/xvdx
 ${var.bastion["nodes"] == 0 && count.index == 0 ? "
 - /tmp/icp_scripts/start_install.sh -i ${local.icp-version} -b ${aws_s3_bucket.icp_config_backup.id} ${local.image_package_uri != "" ? "-c ${local.image_package_uri}" : "" }"
   :
 "" }
 ${var.bastion["nodes"] == 0 && count.index == 0 && var.enable_autoscaling ? "
-- /tmp/icp_scripts/create_client_cert.sh -i ${var.icp_inception_image} -b ${aws_s3_bucket.icp_config_backup.id}"
+- /tmp/icp_scripts/create_client_cert.sh -i ${var.icp_inception_image} -b ${aws_s3_bucket.icp_config_backup.id} -k ${aws_network_interface.mastervip.0.private_ip}"
   :
 "" }
 ${var.master["nodes"] > 1 ? "
diff --git a/scripts/asg-configmap.yaml b/scripts/asg-configmap.yaml
new file mode 100644
index 0000000..a0c7738
--- /dev/null
+++ b/scripts/asg-configmap.yaml
@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: autoscaler-config
+data:
+  add_worker.sh: |
+   #!/bin/bash
+   curl https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -o /tmp/awscli-bundle.zip
+   unzip /tmp/awscli-bundle.zip -d /tmp
+   /tmp/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
+
+   /usr/local/bin/aws s3 cp --no-progress --recursive --exclude "images/*" --exclude "patches/*" s3://${CLUSTER_BACKUP} /installer/cluster
+
+   rm -f /installer/cluster/.install.lock
+   chmod 400 /installer/cluster/ssh_key
+   crudini --set /installer/cluster/hosts worker ${NODE_IP}
+
+   ansible -i /installer/cluster/hosts localhost --private-key /installer/cluster/ssh_key -u icpdeploy -b -m wait_for -a "host=${NODE_IP} port=22 timeout=18000"
+   ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m wait_for -a "path=/var/lib/cloud/instance/boot-finished timeout=18000"
+   echo "Waiting for node to initialize"
+   ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m lineinfile -a 'path=/etc/hosts line="3.13.117.119 icp-console.bluesky.local"'
+   echo ${INSTANCE_NAME} > hostname
+   ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m copy -a "src=hostname dest=/etc/hostname"
+   ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m service -a "name=systemd-hostnamed state=restarted"
+   /usr/local/bin/aws --region ${REGION} ec2 create-tags --resources ${INSTANCEID} --tags Key=Name,Value=${INSTANCE_NAME}
+
+   (
+   /installer/installer.sh worker -l ${NODE_IP} &&
+   /usr/local/bin/aws --region ${REGION} autoscaling complete-lifecycle-action --lifecycle-hook-name ${LIFECYCLEHOOKNAME} --lifecycle-action-token ${LIFECYCLEACTIONTOKEN} --auto-scaling-group-name ${ASGNAME} --lifecycle-action-result CONTINUE --instance-id ${INSTANCEID} &&
+   /usr/local/bin/aws s3 sync /installer/cluster s3://${CLUSTER_BACKUP}
+   ) || (
+   /installer/installer.sh uninstall -l ${NODE_IP}
+   false
+   )
+  remove_worker.sh: |
+   #!/bin/bash
+   curl https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -o /tmp/awscli-bundle.zip
+   unzip /tmp/awscli-bundle.zip -d /tmp
+   /tmp/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
+   aws s3 cp --no-progress --recursive --exclude "images/*" --exclude "patches/*" s3://${CLUSTER_BACKUP} /installer/cluster
+   chmod 400 /installer/cluster/ssh_key
+   crudini --set /installer/cluster/hosts worker ${NODE_IP}
+   rm -f /installer/cluster/.install.lock
+   /installer/installer.sh uninstall -l ${NODE_IP} &&
+   aws --region ${REGION} autoscaling complete-lifecycle-action --lifecycle-hook-name ${LIFECYCLEHOOKNAME} --lifecycle-action-token ${LIFECYCLEACTIONTOKEN} --auto-scaling-group-name ${ASGNAME} --lifecycle-action-result CONTINUE --instance-id ${INSTANCEID} &&
+   crudini --del /installer/cluster/hosts worker ${NODE_IP} &&
+   /usr/local/bin/aws s3 sync /installer/cluster s3://${CLUSTER_BACKUP}
diff --git a/scripts/cluster-autoscaler-rbac.yaml b/scripts/cluster-autoscaler-rbac.yaml
new file mode 100644
index 0000000..ea5836a
--- /dev/null
+++ b/scripts/cluster-autoscaler-rbac.yaml
@@ -0,0 +1,118 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+  name: cluster-autoscaler
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["events", "endpoints"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["pods/eviction"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["pods/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    resourceNames: ["cluster-autoscaler"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["watch", "list", "get", "update"]
+  - apiGroups: [""]
+    resources:
+      - "pods"
+      - "services"
+      - "replicationcontrollers"
+      - "persistentvolumeclaims"
+      - "persistentvolumes"
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["extensions"]
+    resources: ["replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["watch", "list"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets", "replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["batch", "extensions"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups:
+    - policy
+    resourceNames:
+    - ibm-privileged-psp
+    resources:
+    - podsecuritypolicies
+    verbs:
+    - use
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create","list","watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+    verbs: ["delete", "get", "update", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
diff --git a/scripts/create_client_cert.sh b/scripts/create_client_cert.sh
index e133874..ec5d32c 100644
--- a/scripts/create_client_cert.sh
+++ b/scripts/create_client_cert.sh
@@ -2,7 +2,7 @@
 
 source /tmp/icp_scripts/functions.sh
 
-while getopts ":b:i:" arg; do
+while getopts ":b:i:k:" arg; do
     case "${arg}" in
       b)
         s3_lambda_bucket=${OPTARG}
@@ -10,6 +10,9 @@ while getopts ":b:i:" arg; do
       i)
         inception_image=${OPTARG}
         ;;
+      k)
+        kube_master=${OPTARG}
+        ;;
     esac
 done
 
@@ -23,11 +26,41 @@ sudo docker run \
   ${registry}${registry:+/}${org}/${repo}:${tag} \
   cp /usr/local/bin/kubectl /data
 
-/usr/local/bin/kubectl -s localhost:8888 create clusterrolebinding lambda-role --clusterrole=cluster-admin --user=lambda --group=lambda
+/usr/local/bin/kubectl config set-cluster local --server=https://${kube_master}:8001 --insecure-skip-tls-verify=true
+/usr/local/bin/kubectl config set-credentials user --embed-certs=true --client-certificate=/opt/ibm/cluster/cfc-certs/kubernetes/kubecfg.crt --client-key=/opt/ibm/cluster/cfc-certs/kubernetes/kubecfg.key
+/usr/local/bin/kubectl config set-context ctx --cluster=local --user=user --namespace=kube-system
+/usr/local/bin/kubectl config use-context ctx
+
+/usr/local/bin/kubectl create clusterrolebinding lambda-role --clusterrole=cluster-admin --user=lambda --group=lambda
+/usr/local/bin/kubectl -n default apply -f /tmp/icp_scripts/asg-configmap.yaml
+
 openssl genrsa -out /tmp/lambda-key.pem 4096
 openssl req -new -key /tmp/lambda-key.pem -out /tmp/lambda-cert.csr -subj '/O=lambda/CN=lambda'
-openssl x509 -req -days 3650 -sha256 -in /tmp/lambda-cert.csr -CA /etc/cfc/conf/ca.crt -CAkey /etc/cfc/conf/ca.key -set_serial 2 -out /tmp/lambda-cert.pem
+openssl x509 -req -days 3650 -sha256 -in /tmp/lambda-cert.csr -CA /opt/ibm/cluster/cfc-certs/root-ca/ca.crt -CAkey /opt/ibm/cluster/cfc-certs/root-ca/ca.key -set_serial 2 -out /tmp/lambda-cert.pem
 
 /usr/local/bin/aws s3 cp /tmp/lambda-cert.pem s3://${s3_lambda_bucket}/lambda-cert.pem
 /usr/local/bin/aws s3 cp /tmp/lambda-key.pem s3://${s3_lambda_bucket}/lambda-key.pem
-/usr/local/bin/aws s3 cp /etc/cfc/conf/ca.crt s3://${s3_lambda_bucket}/ca.crt
+/usr/local/bin/aws s3 cp /opt/ibm/cluster/cfc-certs/root-ca/ca.crt s3://${s3_lambda_bucket}/ca.crt
+
+cat <<EOF > ~/kuberc
+export PATH=$PATH:/usr/local/bin
+kubectl config set-cluster local --server=https://${kube_master}:8001 --insecure-skip-tls-verify=true
+kubectl config set-credentials user --embed-certs=true --client-certificate=/opt/ibm/cluster/cfc-certs/kubernetes/kubecfg.crt --client-key=/opt/ibm/cluster/cfc-certs/kubernetes/kubecfg.key
+kubectl config set-context ctx --cluster=local --user=user --namespace=kube-system
+kubectl config use-context ctx
+EOF
+
+/usr/local/bin/kubectl get secret infra-registry-key -o yaml | grep -v annotations | grep -v last-applied-configuration | grep -v creationTimestamp | grep -v namespace | grep -v resourceVersion | grep -v uid | /usr/local/bin/kubectl -n default apply -f -
+
+/usr/local/bin/kubectl -n default patch serviceaccount default -p '{"imagePullSecrets": [{"name": "infra-registry-key"}]}'
+
+/usr/local/bin/kubectl -n kube-system apply -f /tmp/icp_scripts/cluster-autoscaler-rbac.yaml
+
+/usr/local/bin/kubectl -n kube-system apply -f /tmp/icp_scripts/cluster-autoscaler-deployment.yaml
+
+# Cluster autoscaler recommends running same version as kube version, however CA versions only started to sync with kube version starting with 1.12.x
+# this command below should manage earlier versions of kube
+KUBE_VERSION=$(/usr/local/bin/kubectl version -o json | python -c "import sys, json; print json.load(sys.stdin)['serverVersion']['gitVersion'].split('+')[0].replace('v1.11.','v1.3.').replace('v1.10.','v1.2.').replace('v1.9.','v1.1.').replace('v1.8.','v1.0.')")
+
+/usr/local/bin/kubectl -n kube-system set image deployment/cluster-autoscaler cluster-autoscaler=k8s.gcr.io/cluster-autoscaler:${KUBE_VERSION}
+
diff --git a/scripts/start_install.sh b/scripts/start_install.sh
index 215684c..e7c2a0b 100644
--- a/scripts/start_install.sh
+++ b/scripts/start_install.sh
@@ -3,6 +3,8 @@
 source /tmp/icp_scripts/functions.sh
 
 logfile="/tmp/icp_logs/start_install.log"
+mkdir -p /tmp/icp_logs
+
 
 #
 # Function for logging output.
@@ -64,7 +66,7 @@ if [[ ! -z "${image_location}" ]]; then
   if [[ "${image_location:0:2}" == "s3" ]]; then
     # stream it right out of s3 into docker
     logmsg "Copying binary package from ${image_location} ..."
-    ${awscli} s3 cp ${image_location} /tmp --no-progress
+    ${awscli} s3 cp --no-progress ${image_location} /tmp 
 
     logmsg "Loading docker images from /tmp/`basename ${image_location}` ..."
     tar zxf /tmp/`basename ${image_location}` -O | docker load | tee -a $logfile
@@ -92,19 +94,16 @@ docker run -v `pwd`:/deploy -w=/deploy --entrypoint=git hashicorp/terraform:0.11
 docker run -v `pwd`:/deploy -w=/deploy/terraform-module-icp-deploy --entrypoint=git hashicorp/terraform:0.11.14 checkout 3.1.1
 
 # write the terraform.tfvars
-${awscli} s3 cp s3://${s3_config_bucket}/terraform.tfvars terraform-module-icp-deploy/terraform.tfvars
+${awscli} s3 cp --no-progress s3://${s3_config_bucket}/terraform.tfvars terraform-module-icp-deploy/terraform.tfvars
 
 # write the additional icp config file for merging
-${awscli} s3 cp s3://${s3_config_bucket}/icp-terraform-config.yaml terraform-module-icp-deploy/icp-terraform-config.yaml
+${awscli} s3 cp --no-progress s3://${s3_config_bucket}/icp-terraform-config.yaml terraform-module-icp-deploy/icp-terraform-config.yaml
 
 docker run -v `pwd`:/deploy -w=/deploy/terraform-module-icp-deploy hashicorp/terraform:0.11.14 init
 docker run -v `pwd`:/deploy -w=/deploy/terraform-module-icp-deploy hashicorp/terraform:0.11.14 apply -auto-approve
-instretval=$?
 
 # backup the config
 logmsg "Backing up the config to the S3 bucket."
-${awscli} s3 sync /opt/ibm/cluster s3://${s3_config_bucket} --no-progress
+${awscli} s3 sync /opt/ibm/cluster s3://${s3_config_bucket}
 
 logmsg "~~~~~~~~ Completed ICP installation Code ~~~~~~~~"
-# Ensure the script exits with the exit code of the ICP installer
-exit ${instretval}

From 60f166e1855914e3f62e0ef0c54150e50dc7b2b6 Mon Sep 17 00:00:00 2001
From: emk <nh43499@hertz.net>
Date: Wed, 28 Aug 2019 15:17:59 +0100
Subject: [PATCH 2/2] cleanup asg-configmap.yaml

---
 scripts/asg-configmap.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/asg-configmap.yaml b/scripts/asg-configmap.yaml
index a0c7738..b0eecf6 100644
--- a/scripts/asg-configmap.yaml
+++ b/scripts/asg-configmap.yaml
@@ -18,7 +18,6 @@ data:
    ansible -i /installer/cluster/hosts localhost --private-key /installer/cluster/ssh_key -u icpdeploy -b -m wait_for -a "host=${NODE_IP} port=22 timeout=18000"
    ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m wait_for -a "path=/var/lib/cloud/instance/boot-finished timeout=18000"
    echo "Waiting for node to initialize"
-   ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m lineinfile -a 'path=/etc/hosts line="3.13.117.119 icp-console.bluesky.local"'
    echo ${INSTANCE_NAME} > hostname
    ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m copy -a "src=hostname dest=/etc/hostname"
    ansible -i /installer/cluster/hosts ${NODE_IP} --private-key /installer/cluster/ssh_key -u icpdeploy -b -m service -a "name=systemd-hostnamed state=restarted"