Skip to content

Commit 2ead364

Browse files
author
emk
committed
updates for cluster autoscaler functionality. fixed issues with index.js and common.js. updated lambda function to nodejs8.10. included deployment of kubernetes/autoscaler deployment. created configmap for install and delete scripts to remove from javascript.
1 parent df4d950 commit 2ead364

15 files changed

+493
-104
lines changed

‎autoscaling.tf

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,17 @@ module "icpautoscaling" {
2424
#icpuser = "aws_lb_target_group_attachment.master-8001.arn" // attempt at workaround for missing depends on
2525

2626
kube_api_url = "https://${aws_lb.icp-console.dns_name}:8001"
27+
docker_registry = "${var.user_provided_cert_dns != "" ? var.user_provided_cert_dns : aws_lb.icp-console.dns_name}:8500"
2728

2829
aws_region = "${var.aws_region}"
2930
azs = ["${var.azs}"]
30-
ami = "${var.worker["ami"] != "" ? var.worker["ami"] : lookup(local.default_searched_ami, var.ami, var.ami)}"
31+
ami = "${var.worker["ami"] != "" ? var.worker["ami"] : local.default_ami }"
3132
worker_root_disk_size = "${var.worker["disk"]}"
3233
worker_docker_vol_size = "${var.worker["docker_vol"]}"
3334
key_name = "${var.key_name}"
3435
instance_type = "${var.worker["type"]}"
36+
ebs_optimized = "${var.worker["ebs_optimized"]}"
37+
instance_name = "${var.instance_name}"
3538
security_groups = [
3639
"${aws_security_group.default.id}"
3740
]
@@ -44,4 +47,90 @@ module "icpautoscaling" {
4447
image_location = "${local.image_package_uri}"
4548
icp_inception_image = "${var.icp_inception_image}"
4649
lambda_s3_bucket = "${local.lambda_s3_bucket}"
50+
icp_config_s3_bucket = "${aws_s3_bucket.icp_config_backup.id}"
51+
asg_tags = ["${data.null_data_source.asg-tags.*.outputs}"]
4752
}
53+
54+
data "null_data_source" "asg-tags" {
55+
count = "${length(keys(var.default_tags))}"
56+
inputs = {
57+
key = "${element(keys(var.default_tags), count.index)}"
58+
value = "${element(values(var.default_tags), count.index)}"
59+
propagate_at_launch = "true"
60+
}
61+
}
62+
63+
resource "aws_s3_bucket_object" "icp_cluster_autoscaler_yaml" {
64+
bucket = "${aws_s3_bucket.icp_config_backup.id}"
65+
key = "scripts/cluster-autoscaler-deployment.yaml"
66+
content = <<EOF
67+
apiVersion: apps/v1
68+
kind: Deployment
69+
metadata:
70+
name: cluster-autoscaler
71+
namespace: kube-system
72+
labels:
73+
app: cluster-autoscaler
74+
spec:
75+
replicas: 1
76+
selector:
77+
matchLabels:
78+
app: cluster-autoscaler
79+
template:
80+
metadata:
81+
labels:
82+
app: cluster-autoscaler
83+
spec:
84+
serviceAccountName: cluster-autoscaler
85+
containers:
86+
- image: k8s.gcr.io/cluster-autoscaler:v1.2.2
87+
name: cluster-autoscaler
88+
resources:
89+
limits:
90+
cpu: 100m
91+
memory: 300Mi
92+
requests:
93+
cpu: 100m
94+
memory: 300Mi
95+
command:
96+
- ./cluster-autoscaler
97+
- --v=4
98+
- --stderrthreshold=info
99+
- --cloud-provider=aws
100+
- --skip-nodes-with-local-storage=false
101+
- --skip-nodes-with-system-pods=false
102+
- --expander=least-waste
103+
- --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,kubernetes.io/cluster/${random_id.clusterid.hex}
104+
- --balance-similar-node-groups=true
105+
volumeMounts:
106+
- name: ssl-certs
107+
mountPath: /etc/ssl/certs/ca-certificates.crt
108+
readOnly: true
109+
imagePullPolicy: "Always"
110+
nodeSelector:
111+
master: "true"
112+
tolerations:
113+
- effect: NoSchedule
114+
key: dedicated
115+
operator: Exists
116+
- key: CriticalAddonsOnly
117+
operator: Exists
118+
volumes:
119+
- name: ssl-certs
120+
hostPath:
121+
path: "/etc/ssl/certs/ca-bundle.crt"
122+
EOF
123+
}
124+
125+
resource "aws_s3_bucket_object" "asg_configmap" {
126+
bucket = "${aws_s3_bucket.icp_config_backup.id}"
127+
key = "scripts/asg-configmap.yaml"
128+
source = "${path.module}/scripts/asg-configmap.yaml"
129+
}
130+
131+
resource "aws_s3_bucket_object" "cluster_autoscaler_rbac_yaml" {
132+
bucket = "${aws_s3_bucket.icp_config_backup.id}"
133+
key = "scripts/cluster-autoscaler-rbac.yaml"
134+
source = "${path.module}/scripts/cluster-autoscaler-rbac.yaml"
135+
}
136+

‎autoscaling/autoscalinggroup.tf

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ resource "aws_launch_configuration" "icp_worker_lc" {
1010

1111
security_groups = ["${var.security_groups}"]
1212

13-
ebs_optimized = true
13+
ebs_optimized = "${var.ebs_optimized}"
1414
root_block_device {
1515
volume_size = "${var.worker_root_disk_size}"
1616
}
@@ -27,15 +27,17 @@ resource "aws_launch_configuration" "icp_worker_lc" {
2727
packages:
2828
- unzip
2929
- python
30+
- bind-utils
3031
rh_subscription:
3132
enable-repo: rhui-REGION-rhel-server-optional
3233
write_files:
33-
- path: /tmp/bootstrap.sh
34+
- path: /tmp/bootstrap-node.sh
3435
permissions: '0755'
3536
encoding: b64
36-
content: ${base64encode(file("${path.module}/../scripts/bootstrap.sh"))}
37+
content: ${base64encode(file("${path.module}/../scripts/bootstrap-node.sh"))}
3738
runcmd:
38-
- /tmp/bootstrap.sh ${var.docker_package_location != "" ? "-p ${var.docker_package_location}" : "" } -d /dev/xvdx ${var.image_location != "" ? "-i ${var.image_location}" : "" } -s ${var.icp_inception_image}
39+
- /tmp/bootstrap-node.sh -c ${var.icp_config_s3_bucket} -s "bootstrap.sh"
40+
- /tmp/icp_scripts/bootstrap.sh ${var.docker_package_location != "" ? "-p ${var.docker_package_location}" : "" } -d /dev/xvdx ${var.image_location != "" ? "-i ${var.image_location}" : "" } -s ${var.icp_inception_image}
3941
users:
4042
- default
4143
- name: icpdeploy
@@ -54,57 +56,58 @@ EOF
5456
}
5557

5658
resource "aws_autoscaling_group" "icp_worker_asg" {
57-
count = "${var.enabled ? 1 : 0}"
58-
name = "icp-worker-asg-${var.cluster_id}"
59+
count = "${length(var.azs)}"
60+
name = "icp-worker-asg-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
5961
launch_configuration = "${aws_launch_configuration.icp_worker_lc.name}"
6062
min_size = 0
6163
max_size = 20
6264
force_delete = true
6365

64-
availability_zones = "${formatlist("%v%v", var.aws_region, var.azs)}"
65-
vpc_zone_identifier = ["${var.private_subnet_ids}"]
66+
vpc_zone_identifier = ["${element(var.private_subnet_ids, count.index)}"]
6667

67-
tags = [
68-
{
69-
key = "kubernetes.io/cluster/${var.cluster_id}",
70-
value = "${var.cluster_id}",
71-
propagate_at_launch = true
72-
}
73-
]
68+
tags = ["${concat(
69+
var.asg_tags,
70+
list(map("key", "k8s.io/cluster-autoscaler/enabled", "value", "${var.enabled}", "propagate_at_launch", "false")),
71+
list(map("key", "kubernetes.io/cluster/${var.cluster_id}", "value", "${var.cluster_id}", "propagate_at_launch", "true"))
72+
)}"]
7473
}
7574

7675
resource "aws_autoscaling_lifecycle_hook" "icp_add_worker_hook" {
77-
count = "${var.enabled ? 1 : 0}"
78-
name = "icp-workernode-added-${var.cluster_id}"
79-
autoscaling_group_name = "${aws_autoscaling_group.icp_worker_asg.name}"
76+
count = "${length(var.azs)}"
77+
name = "icp-workernode-added-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
78+
autoscaling_group_name = "${element(aws_autoscaling_group.icp_worker_asg.*.name, count.index)}"
8079
default_result = "ABANDON"
8180
heartbeat_timeout = 3600
8281
lifecycle_transition = "autoscaling:EC2_INSTANCE_LAUNCHING"
8382

8483
notification_metadata = <<EOF
8584
{
86-
"icp_inception_image": "${var.icp_inception_image}",
85+
"icp_inception_image": "${var.docker_registry}/${var.icp_inception_image}",
8786
"docker_package_location": "${var.docker_package_location}",
8887
"image_location": "${var.image_location}",
89-
"cluster_backup": "icpbackup-${var.cluster_id}"
88+
"cluster_backup": "icpbackup-${var.cluster_id}",
89+
"cluster_id": "${var.cluster_id}",
90+
"instance_name": "${var.instance_name}"
9091
}
9192
EOF
9293
}
9394

9495
resource "aws_autoscaling_lifecycle_hook" "icp_del_worker_hook" {
95-
count = "${var.enabled ? 1 : 0}"
96-
name = "icp-workernode-removed-${var.cluster_id}"
97-
autoscaling_group_name = "${aws_autoscaling_group.icp_worker_asg.name}"
96+
count = "${length(var.azs)}"
97+
name = "icp-workernode-removed-${var.aws_region}${element(var.azs, count.index)}-${var.cluster_id}"
98+
autoscaling_group_name = "${element(aws_autoscaling_group.icp_worker_asg.*.name, count.index)}"
9899
default_result = "ABANDON"
99100
heartbeat_timeout = 3600
100101
lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING"
101102

102103
notification_metadata = <<EOF
103104
{
104-
"icp_inception_image": "${var.icp_inception_image}",
105+
"icp_inception_image": "${var.docker_registry}/${var.icp_inception_image}",
105106
"docker_package_location": "${var.docker_package_location}",
106107
"image_location": "${var.image_location}",
107-
"cluster_backup": "icpbackup-${var.cluster_id}"
108+
"cluster_backup": "icpbackup-${var.cluster_id}",
109+
"cluster_id": "${var.cluster_id}",
110+
"instance_name": "${var.instance_name}"
108111
}
109112
EOF
110113
}

‎autoscaling/cloudwatch.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ resource "aws_cloudwatch_event_rule" "icp_worker_node_added_event" {
1313
],
1414
"detail": {
1515
"AutoScalingGroupName": [
16-
"${aws_autoscaling_group.icp_worker_asg.name}"
16+
${join(",", formatlist("\"%v\"", aws_autoscaling_group.icp_worker_asg.*.name))}
1717
]
1818
}
1919
}
@@ -35,7 +35,7 @@ resource "aws_cloudwatch_event_rule" "icp_worker_node_remove_event" {
3535
],
3636
"detail": {
3737
"AutoScalingGroupName": [
38-
"${aws_autoscaling_group.icp_worker_asg.name}"
38+
${join(",", formatlist("\"%v\"", aws_autoscaling_group.icp_worker_asg.*.name))}
3939
]
4040
}
4141
}

‎autoscaling/lambda.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ resource "aws_lambda_function" "icp_autoscale" {
4747
function_name = "icp-worker-autoscale-${var.cluster_id}"
4848
role = "${local.iam_lambda_role_arn}"
4949
handler = "index.handler"
50-
runtime = "nodejs6.10"
50+
runtime = "nodejs8.10"
5151
timeout = 10
5252

5353
vpc_config {

‎autoscaling/lambda/common.js

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,42 @@
11
var aws = require('aws-sdk');
22
var kubeapi = require('kubernetes-client');
33
var fs = require('fs');
4-
4+
var region;
55
aws.config.update({region: region});
66
aws.config.setPromisesDependency(Promise);
77

88
function get_instance_ip(region, instance_id) {
99
var ec2 = new aws.EC2({apiVersion: '2016-11-15'});
1010

11-
console.log("InstanceID: " + instance_id);
11+
console.log("get_instance_ip: Region: " + region);
12+
console.log("get_instance_ip: InstanceID: " + instance_id);
1213

1314
var params = {
14-
DryRun: false,
15-
InstanceIds: [ instance_id ],
15+
//DryRun: false,
16+
InstanceIds: [ instance_id ]
1617
};
1718

18-
return ec2.describeInstances(params, function(err, result) {
19-
if (err) {
20-
console.log(err, err.stack);
19+
var request = ec2.describeInstances(params);
20+
21+
var promise = request.promise();
22+
23+
// handle promise's fulfilled/rejected states
24+
promise.then(
25+
function(data) {
26+
console.log("get_instance_ip: Instance IP address is: " + data.Reservations[0].Instances[0].PrivateIpAddress);
27+
console.log("get_instance_ip: " + JSON.stringify(data, null, 2));
28+
29+
//return data.Reservations[0].Instances[0].PrivateIpAddress;
30+
/* process the data */
31+
},
32+
function(err) {
33+
/* handle the error */
34+
console.log("get_instance_ip: " + err, err.stack);
2135
throw err;
22-
} else {
23-
console.log("Instance IP address is: " + result.Reservations[0].Instances[0].PrivateIpAddress);
24-
return result.Reservations[0].Instances[0].PrivateIpAddress;
2536
}
26-
});
37+
);
38+
39+
return promise;
2740
}
2841

2942
function get_bucket_object(bucketName, key) {
@@ -34,16 +47,26 @@ function get_bucket_object(bucketName, key) {
3447
Key: key
3548
};
3649

37-
s3.getObject(params, function(err, data) {
38-
if (err) {
39-
console.log(err, err.stack);
50+
var request = s3.getObject(params);
51+
52+
var promise = request.promise();
53+
54+
// handle promise's fulfilled/rejected states
55+
promise.then(
56+
function(data) {
57+
console.log("get_bucket_object: body is: " + data.Body);
58+
59+
//return data.Reservations[0].Instances[0].PrivateIpAddress;
60+
/* process the data */
61+
},
62+
function(err) {
63+
/* handle the error */
64+
console.log("get_bucket_object: " + err, err.stack);
4065
throw err;
41-
} else {
42-
console.log(data); // successful response
43-
44-
return data.Body;
4566
}
46-
});
67+
);
68+
69+
return promise;
4770
}
4871

4972
function create_job(ca_crt, client_cert, client_key, job) {
@@ -77,8 +100,10 @@ function fail_autoscaling(params) {
77100
.then(function(result) {
78101
console.log("competed lifecycle action");
79102
});
80-
};
103+
}
81104

82105
module.exports.get_instance_ip = get_instance_ip;
83106
module.exports.create_job = create_job;
84107
module.exports.fail_autoscaling = fail_autoscaling;
108+
module.exports.get_bucket_object = get_bucket_object;
109+

0 commit comments

Comments
 (0)