Skip to content

Commit 218623a

Browse files
tomarv2tomarv2
authored
Merge pull request #60 from tomarv2/develop
Develop
2 parents 3f13861 + 8a5d300 commit 218623a

File tree

44 files changed

+115
-328
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+115
-328
lines changed

‎.pre-commit-config.yaml

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: https://github.com/antonbabenko/pre-commit-terraform
3-
rev: v1.64.0
3+
rev: v1.71.0
44
hooks:
55
- id: terraform_fmt
66
- id: terraform_tflint
@@ -12,7 +12,6 @@ repos:
1212
- '--args=--only=terraform_documented_outputs'
1313
- '--args=--only=terraform_documented_variables'
1414
- '--args=--only=terraform_typed_variables'
15-
#- '--args=--only=terraform_module_pinned_source'
1615
- '--args=--only=terraform_naming_convention'
1716
- '--args=--only=terraform_required_providers'
1817
- '--args=--only=terraform_standard_module_structure'
@@ -29,10 +28,3 @@ repos:
2928
- id: double-quote-string-fixer
3029
- id: name-tests-test
3130
- id: requirements-txt-fixer
32-
33-
- repo: https://github.com/bridgecrewio/checkov.git
34-
rev: '2.0.914'
35-
hooks:
36-
- id: checkov
37-
verbose: true
38-
args: [-d, '.', --framework,'terraform']

‎README.md

Lines changed: 23 additions & 75 deletions
Large diffs are not rendered by default.

‎aws_instance_profile.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
resource "databricks_instance_profile" "shared" {
2+
count = var.deploy_cluster == true && var.add_instance_profile_to_workspace == true ? 1 : 0
3+
4+
instance_profile_arn = lookup(var.aws_attributes, "instance_profile_arn", null)
5+
}

‎cluster.tf

Lines changed: 20 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,12 @@
11
locals {
22
cluster_policy_id = var.cluster_policy_id != null ? var.cluster_policy_id : join("", databricks_cluster_policy.this.*.id)
3-
}
4-
5-
resource "databricks_instance_profile" "shared" {
6-
count = var.deploy_cluster == true && var.add_instance_profile_to_workspace == true ? 1 : 0
7-
8-
instance_profile_arn = lookup(var.aws_attributes, "instance_profile_arn", null)
3+
cluster_name = var.cluster_name != null ? var.cluster_name : "${var.teamid}-${var.prjid} (Terraform managed)"
94
}
105

116
resource "databricks_cluster" "cluster" {
12-
count = var.deploy_cluster == true && (var.fixed_value != 0 || var.auto_scaling != null) ? 1 : 0
7+
for_each = var.deploy_cluster == true ? toset([local.cluster_name]) : toset([])
138

14-
cluster_name = var.cluster_name != null ? var.cluster_name : "${var.teamid}-${var.prjid} (Terraform managed)"
9+
cluster_name = local.cluster_name
1510

1611
policy_id = var.cluster_policy_id == null && var.deploy_cluster_policy == false ? null : local.cluster_policy_id
1712
spark_version = var.spark_version != null ? var.spark_version : data.databricks_spark_version.latest.id
@@ -21,7 +16,7 @@ resource "databricks_cluster" "cluster" {
2116
num_workers = var.fixed_value != null ? var.fixed_value : null
2217

2318
dynamic "autoscale" {
24-
for_each = var.auto_scaling != null ? [var.auto_scaling] : []
19+
for_each = var.fixed_value == null && var.auto_scaling != null ? [var.auto_scaling] : []
2520
content {
2621
min_workers = autoscale.value[0]
2722
max_workers = autoscale.value[1]
@@ -42,46 +37,29 @@ resource "databricks_cluster" "cluster" {
4237
}
4338
}
4439

45-
autotermination_minutes = var.cluster_autotermination_minutes
46-
custom_tags = var.custom_tags != null ? var.custom_tags : null
47-
48-
spark_conf = var.spark_conf
49-
spark_env_vars = var.spark_env_vars
50-
}
51-
52-
resource "databricks_cluster" "single_node_cluster" {
53-
count = var.deploy_cluster == true && var.fixed_value == 0 ? 1 : 0
54-
55-
cluster_name = var.cluster_name != null ? var.cluster_name : "${var.teamid}-${var.prjid} (Terraform managed)"
56-
57-
spark_version = var.spark_version != null ? var.spark_version : data.databricks_spark_version.latest.id
58-
node_type_id = var.deploy_worker_instance_pool != true ? local.driver_node_type : null
59-
num_workers = 0
40+
dynamic "azure_attributes" {
41+
for_each = var.azure_attributes == null ? [] : [var.azure_attributes]
42+
content {
43+
first_on_demand = lookup(azure_attributes.value, "first_on_demand", null)
44+
availability = lookup(azure_attributes.value, "availability", null)
45+
spot_bid_max_price = lookup(azure_attributes.value, "spot_bid_max_price", null)
46+
}
47+
}
6048

61-
dynamic "aws_attributes" {
62-
for_each = var.aws_attributes == null ? [] : [var.aws_attributes]
49+
dynamic "gcp_attributes" {
50+
for_each = var.gcp_attributes == null ? [] : [var.gcp_attributes]
6351
content {
64-
instance_profile_arn = var.add_instance_profile_to_workspace == true ? join("", databricks_instance_profile.shared.*.id) : lookup(aws_attributes.value, "instance_profile_arn", null)
65-
zone_id = lookup(aws_attributes.value, "zone_id", null)
66-
first_on_demand = lookup(aws_attributes.value, "first_on_demand", null)
67-
availability = lookup(aws_attributes.value, "availability", null)
68-
spot_bid_price_percent = lookup(aws_attributes.value, "spot_bid_price_percent", null)
69-
ebs_volume_count = lookup(aws_attributes.value, "ebs_volume_count", null)
70-
ebs_volume_size = lookup(aws_attributes.value, "ebs_volume_size", null)
71-
ebs_volume_type = lookup(aws_attributes.value, "ebs_volume_type", null)
52+
first_on_demand = lookup(gcp_attributes.value, "first_on_demand", null)
53+
availability = lookup(gcp_attributes.value, "availability", null)
54+
spot_bid_max_price = lookup(gcp_attributes.value, "spot_bid_max_price", null)
7255
}
7356
}
7457

7558
autotermination_minutes = var.cluster_autotermination_minutes
59+
custom_tags = var.fixed_value == 0 ? merge({ "ResourceClass" = "SingleNode" }, var.custom_tags) : var.custom_tags
7660

77-
custom_tags = {
78-
"ResourceClass" = "SingleNode"
79-
}
80-
spark_conf = {
81-
# Single-node
82-
"spark.databricks.cluster.profile" : "singleNode"
83-
"spark.master" : "local[*]"
84-
}
61+
spark_conf = var.fixed_value == 0 ? { "spark.databricks.cluster.profile" : "singleNode"
62+
"spark.master" : "local[*]" } : var.spark_conf
8563

8664
spark_env_vars = var.spark_env_vars
8765
}

‎examples/all/sample-maximum_config/main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}

‎examples/all/sample-maximum_config/variables.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
variable "teamid" {
2-
description = "(Required) Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
2+
description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
33
type = string
44
}
55

66
variable "prjid" {
7-
description = "(Required) Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
7+
description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
88
type = string
99
}
1010

‎examples/all/sample/main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}

‎examples/all/sample/variables.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
variable "teamid" {
2-
description = "(Required) Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
2+
description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
33
type = string
44
}
55

66
variable "prjid" {
7-
description = "(Required) Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
7+
description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
88
type = string
99
}
1010

‎examples/clusters/1_single_node_cluster/main.tf

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,13 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}
1414

1515
module "databricks_workspace_management" {
1616
source = "../../../"
17-
18-
workspace_url = var.workspace_url
19-
dapi_token = var.dapi_token
2017
# ------------------------------------------------
2118
# CLUSTER
2219
# ------------------------------------------------

‎examples/clusters/1_single_node_cluster/outputs.tf

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,6 @@ output "cluster_id" {
33
value = module.databricks_workspace_management.cluster_id
44
}
55

6-
output "standalone_cluster_id" {
7-
description = "databricks standalone cluster id"
8-
value = module.databricks_workspace_management.single_node_cluster_id
9-
}
10-
116
output "databricks_secret_acl" {
127
description = "databricks secret acl"
138
value = module.databricks_workspace_management.databricks_secret_acl

‎examples/clusters/1_single_node_cluster/variables.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
variable "teamid" {
2-
description = "(Required) Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
2+
description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
33
type = string
44
}
55

66
variable "prjid" {
7-
description = "(Required) Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
7+
description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
88
type = string
99
}
1010

‎examples/clusters/2_standard_cluster/main.tf

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,29 +7,26 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}
1414

1515

1616
module "databricks_workspace_management" {
1717
source = "../../../"
18-
19-
workspace_url = var.workspace_url
20-
dapi_token = var.dapi_token
2118
# ------------------------------------------------
2219
# CLUSTER
2320
# ------------------------------------------------
2421
deploy_cluster = true
2522
fixed_value = 1
2623
cluster_autotermination_minutes = 30
27-
worker_node_type_id = "i3.xlarge"
28-
driver_node_type_id = "i3.xlarge"
24+
worker_node_type_id = "Standard_DS3_v2"
25+
driver_node_type_id = "Standard_DS3_v2"
2926
spark_version = "8.3.x-scala2.12"
3027
spark_conf = {
3128
"spark.databricks.io.cache.enabled" = true
32-
"spark.driver.maxResultSize" = "10g"
29+
"spark.driver.maxResultSize" = "20g"
3330
}
3431
# ------------------------------------------------
3532
# Do not change the teamid, prjid once set.

‎examples/clusters/2_standard_cluster/outputs.tf

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,6 @@ output "cluster_id" {
33
value = module.databricks_workspace_management.cluster_id
44
}
55

6-
output "standalone_cluster_id" {
7-
description = "databricks standalone cluster id"
8-
value = module.databricks_workspace_management.single_node_cluster_id
9-
}
10-
116
output "databricks_secret_acl" {
127
description = "databricks secret acl"
138
value = module.databricks_workspace_management.databricks_secret_acl

‎examples/clusters/2_standard_cluster/variables.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
variable "teamid" {
2-
description = "(Required) Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
2+
description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
33
type = string
44
}
55

66
variable "prjid" {
7-
description = "(Required) Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
7+
description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
88
type = string
99
}
1010

‎examples/clusters/3_standard_cluster_with_auto_scaling/main.tf

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,24 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}
1414

1515

1616
module "databricks_workspace_management" {
1717
source = "../../../"
18-
19-
workspace_url = var.workspace_url
20-
dapi_token = var.dapi_token
2118
# ------------------------------------------------
2219
# CLUSTER
2320
# ------------------------------------------------
2421
deploy_cluster = true
25-
auto_scaling = [1, 3]
22+
23+
auto_scaling = [3, 5]
24+
25+
custom_tags = {
26+
"Application" : "DemoApp"
27+
}
2628
# ------------------------------------------------
2729
# Do not change the teamid, prjid once set.
2830
teamid = var.teamid

‎examples/clusters/3_standard_cluster_with_auto_scaling/outputs.tf

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,6 @@ output "cluster_id" {
33
value = module.databricks_workspace_management.cluster_id
44
}
55

6-
output "standalone_cluster_id" {
7-
description = "databricks standalone cluster id"
8-
value = module.databricks_workspace_management.single_node_cluster_id
9-
}
10-
116
output "databricks_secret_acl" {
127
description = "databricks secret acl"
138
value = module.databricks_workspace_management.databricks_secret_acl

‎examples/clusters/3_standard_cluster_with_auto_scaling/variables.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
variable "teamid" {
2-
description = "(Required) Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
2+
description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
33
type = string
44
}
55

66
variable "prjid" {
7-
description = "(Required) Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
7+
description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
88
type = string
99
}
1010

‎examples/clusters/4_cluster_with_new_cluster_policy_and_permissions/main.tf

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,13 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}
1414

1515
module "databricks_workspace_management" {
1616
source = "../../../"
17-
18-
workspace_url = var.workspace_url
19-
dapi_token = var.dapi_token
2017
# ------------------------------------------------
2118
# CLUSTER
2219
# ------------------------------------------------

‎examples/clusters/4_cluster_with_new_cluster_policy_and_permissions/outputs.tf

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,6 @@ output "cluster_id" {
33
value = module.databricks_workspace_management.cluster_id
44
}
55

6-
output "standalone_cluster_id" {
7-
description = "databricks standalone cluster id"
8-
value = module.databricks_workspace_management.single_node_cluster_id
9-
}
10-
116
output "databricks_secret_acl" {
127
description = "databricks secret acl"
138
value = module.databricks_workspace_management.databricks_secret_acl

‎examples/clusters/4_cluster_with_new_cluster_policy_and_permissions/variables.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
variable "teamid" {
2-
description = "(Required) Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
2+
description = "Name of the team/group e.g. devops, dataengineering. Should not be changed after running 'tf apply'"
33
type = string
44
}
55

66
variable "prjid" {
7-
description = "(Required) Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
7+
description = "Name of the project/stack e.g: mystack, nifieks, demoaci. Should not be changed after running 'tf apply'"
88
type = string
99
}
1010

‎examples/clusters/5_cluster_with_existing_cluster_policy/main.tf

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,14 @@ terraform {
77
required_providers {
88
databricks = {
99
source = "databrickslabs/databricks"
10-
version = "~> 0.3.5"
10+
version = "~> 0.5.7"
1111
}
1212
}
1313
}
1414

1515

1616
module "databricks_workspace_management" {
1717
source = "../../../"
18-
19-
workspace_url = var.workspace_url
20-
dapi_token = var.dapi_token
2118
# ------------------------------------------------
2219
# CLUSTER
2320
# ------------------------------------------------

0 commit comments

Comments
 (0)