Skip to content

Commit 7692083

Browse files
tomarv2tomarv2
committed
adding support for jobs clusters
1 parent 83642d3 commit 7692083

File tree

11 files changed

+119
-133
lines changed

11 files changed

+119
-133
lines changed

‎.github/workflows/onrelease.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,20 @@ on:
77
name: release
88

99
jobs:
10+
generate-changelog:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v2
14+
with:
15+
fetch-depth: 0
16+
- uses: BobAnkh/auto-generate-changelog@master
17+
with:
18+
REPO_NAME: ''
19+
ACCESS_TOKEN: ${{secrets.GITHUB_TOKEN}}
20+
BRANCH: ''
21+
PATH: 'CHANGELOG.md'
22+
COMMIT_MESSAGE: 'docs(CHANGELOG): update release notes'
23+
TYPE: 'feat:Feature,fix:Bug Fixes,docs:Documentation,refactor:Refactor,perf:Performance Improvements'
1024
build:
1125
name: Create Release
1226
runs-on: ubuntu-latest

‎.pre-commit-config.yaml

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,17 @@ repos:
33
rev: v1.48.0
44
hooks:
55
- id: terraform_fmt
6-
# - id: terraform_validate
76
- id: terraform_tflint
87
args:
98
- '--args=--only=terraform_deprecated_interpolation'
109
- '--args=--only=terraform_deprecated_index'
11-
# - '--args=--only=terraform_unused_declarations'
1210
- '--args=--only=terraform_comment_syntax'
1311
- '--args=--only=terraform_documented_outputs'
1412
- '--args=--only=terraform_documented_variables'
1513
- '--args=--only=terraform_typed_variables'
1614
- '--args=--only=terraform_module_pinned_source'
1715
- '--args=--only=terraform_naming_convention'
1816
- '--args=--only=terraform_required_providers'
19-
# - '--args=--only=terraform_standard_module_structure'
2017
- '--args=--only=terraform_workspace_remote'
2118

2219
- repo: https://github.com/pre-commit/pre-commit-hooks
@@ -28,9 +25,8 @@ repos:
2825
- id: check-yaml
2926

3027
- repo: https://github.com/bridgecrewio/checkov.git
31-
rev: '1.0.864' # change to tag or sha
28+
rev: '2.0.469'
3229
hooks:
3330
- id: checkov
3431
verbose: true
35-
args:
36-
- -d . --framework terraform -o output_format json
32+
args: [-d, '.', --framework,'terraform']

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
## Versions
2929

3030
- Module tested for Terraform 1.0.1.
31-
- `databrickslabs/databricks` provider version [0.3.5](https://registry.terraform.io/providers/databrickslabs/databricks/latest)
32-
- AWS provider version [3.47.0](https://registry.terraform.io/providers/hashicorp/aws/latest).
31+
- `databrickslabs/databricks` provider version [0.3.9](https://registry.terraform.io/providers/databrickslabs/databricks/latest)
32+
- AWS provider version [3.61](https://registry.terraform.io/providers/hashicorp/aws/latest).
3333
- `main` branch: Provider versions not pinned to keep up with Terraform releases.
3434
- `tags` releases: Tags are pinned with versions (use <a href="https://github.com/tomarv2/terraform-databricks-workspace-management/tags" alt="GitHub tag">
3535
<img src="https://img.shields.io/github/v/tag/tomarv2/terraform-databricks-workspace-management" /></a>).

‎cluster.tf

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ locals {
33
}
44

55
resource "databricks_instance_profile" "shared" {
6-
count = (var.deploy_cluster == true && var.add_instance_profile_to_workspace == true) ? 1 : 0
6+
count = var.deploy_cluster == true && var.add_instance_profile_to_workspace == true ? 1 : 0
77

88
instance_profile_arn = lookup(var.aws_attributes, "instance_profile_arn", null)
99
}
1010

1111
resource "databricks_cluster" "cluster" {
12-
count = (var.deploy_cluster == true && (var.fixed_value != 0 || var.auto_scaling != null) ? 1 : 0)
12+
count = var.deploy_cluster == true && (var.fixed_value != 0 || var.auto_scaling != null) ? 1 : 0
1313

1414
cluster_name = var.cluster_name != null ? var.cluster_name : "${var.teamid}-${var.prjid} (Terraform managed)"
1515

@@ -42,14 +42,13 @@ resource "databricks_cluster" "cluster" {
4242
}
4343

4444
autotermination_minutes = var.cluster_autotermination_minutes
45-
#custom_tags = merge(local.shared_tags)
46-
custom_tags = var.custom_tags != null ? merge(var.custom_tags, local.shared_tags) : merge(local.shared_tags)
45+
custom_tags = var.custom_tags != null ? merge(var.custom_tags, local.shared_tags) : merge(local.shared_tags)
4746

48-
spark_conf = var.spark_conf
47+
spark_conf = var.spark_conf
4948
}
5049

5150
resource "databricks_cluster" "single_node_cluster" {
52-
count = var.deploy_cluster == true && var.fixed_value == 0 && var.auto_scaling == null ? 1 : 0
51+
count = var.deploy_cluster == true && var.fixed_value == 0 ? 1 : 0
5352

5453
cluster_name = var.cluster_name != null ? var.cluster_name : "${var.teamid}-${var.prjid} (Terraform managed)"
5554

‎cluster_node_type.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
data "databricks_node_type" "cluster_node_type" {
2-
count = var.deploy_cluster ? 1 : 0
2+
count = var.deploy_cluster || var.deploy_job_cluster ? 1 : 0
33

44
local_disk = var.local_disk
55
min_cores = var.min_cores

‎examples/all/sample-maximum_config/output.tf

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,11 @@ output "cluster_id" {
33
value = module.databricks_workspace_management.cluster_id
44
}
55

6-
output "databricks_job_url" {
7-
description = "databricks job url"
8-
value = module.databricks_workspace_management.job_url
9-
}
10-
116
output "databricks_notebook_url" {
127
description = "databricks notebook url"
138
value = module.databricks_workspace_management.notebook_url
149
}
1510

16-
output "job_new_cluster_url" {
17-
description = "databricks new cluster job url"
18-
value = module.databricks_workspace_management.job_new_cluster_url
19-
}
20-
2111
output "databricks_secret_acl" {
2212
description = "databricks secret acl"
2313
value = module.databricks_workspace_management.databricks_secret_acl
@@ -42,33 +32,3 @@ output "databricks_group_member" {
4232
description = "databricks group members"
4333
value = module.databricks_workspace_management.databricks_group_member
4434
}
45-
46-
output "notebook_permissions" {
47-
description = "databricks notebook permissions"
48-
value = module.databricks_workspace_management.notebook_permissions
49-
}
50-
51-
output "job_permissions" {
52-
description = "databricks job permissions"
53-
value = module.databricks_workspace_management.job_permissions
54-
}
55-
56-
output "cluster_permissions" {
57-
description = "databricks cluster permissions"
58-
value = module.databricks_workspace_management.cluster_permissions
59-
}
60-
61-
output "cluster_policy_permissions" {
62-
description = "databricks cluster policy permissions"
63-
value = module.databricks_workspace_management.cluster_policy_permissions
64-
}
65-
66-
output "driver_pool_permissions" {
67-
description = "databricks driver pool permissions"
68-
value = module.databricks_workspace_management.driver_pool_permissions
69-
}
70-
71-
output "worker_pool_permissions" {
72-
description = "databricks worker pool permissions"
73-
value = module.databricks_workspace_management.driver_pool_permissions
74-
}

‎examples/all/sample/output.tf

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -3,28 +3,11 @@ output "cluster_id" {
33
value = module.databricks_workspace_management.cluster_id
44
}
55

6-
/*
7-
output "databricks_token" {
8-
description = "databricks token"
9-
value = module.databricks_workspace_management.databricks_token
10-
}
11-
*/
12-
13-
output "databricks_job_url" {
14-
description = "databricks job url"
15-
value = module.databricks_workspace_management.job_url
16-
}
17-
186
output "databricks_notebook_url" {
197
description = "databricks notebook url"
208
value = module.databricks_workspace_management.notebook_url
219
}
2210

23-
output "job_new_cluster_url" {
24-
description = "databricks new cluster job url"
25-
value = module.databricks_workspace_management.job_new_cluster_url
26-
}
27-
2811
output "databricks_secret_acl" {
2912
description = "databricks secret acl"
3013
value = module.databricks_workspace_management.databricks_secret_acl
@@ -49,29 +32,3 @@ output "databricks_group_member" {
4932
description = "databricks group members"
5033
value = module.databricks_workspace_management.databricks_group_member
5134
}
52-
53-
output "databricks_permissions_notebook" {
54-
description = "databricks notebook permissions"
55-
value = module.databricks_workspace_management.databricks_permissions_notebook
56-
}
57-
58-
/*
59-
output "databricks_permissions_job" {
60-
value = module.databricks_workspace_management.databricks_permissions_job
61-
}
62-
*/
63-
64-
output "databricks_permissions_cluster" {
65-
description = "databricks cluster permissions"
66-
value = module.databricks_workspace_management.databricks_permissions_cluster
67-
}
68-
69-
output "databricks_permissions_policy" {
70-
description = "databricks cluster policy"
71-
value = module.databricks_workspace_management.databricks_permissions_policy
72-
}
73-
74-
output "databricks_permissions_pool" {
75-
description = "databricks instance pool permissions"
76-
value = module.databricks_workspace_management.databricks_permissions_pool
77-
}

‎job.tf

Lines changed: 82 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,43 @@
33
# 1. NEW CLUSTER WITH NEW NOTEBOOKS
44
# ------------------------------------------------
55
resource "databricks_job" "new_cluster_new_job_new_notebooks" {
6-
for_each = (var.deploy_jobs == true && var.cluster_id == null && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
6+
for_each = (var.deploy_jobs == true && var.cluster_id == null && var.deploy_job_cluster == true && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
77

88
name = "${each.value.job_name} (Terraform managed)"
99

1010
new_cluster {
11-
num_workers = var.num_workers
12-
spark_version = data.databricks_spark_version.latest.id
13-
node_type_id = join("", data.databricks_node_type.cluster_node_type.*.id)
11+
policy_id = var.cluster_policy_id == null && var.deploy_cluster_policy == false ? null : local.cluster_policy_id
12+
spark_version = var.spark_version != null ? var.spark_version : data.databricks_spark_version.latest.id
13+
node_type_id = var.deploy_worker_instance_pool != true ? local.worker_node_type : null
14+
instance_pool_id = var.deploy_worker_instance_pool == true ? join("", databricks_instance_pool.worker_instance_nodes.*.id) : null
15+
driver_node_type_id = var.deploy_worker_instance_pool != true ? local.driver_node_type : null
16+
num_workers = var.fixed_value != null ? var.fixed_value : null
17+
18+
dynamic "autoscale" {
19+
for_each = var.auto_scaling != null ? [var.auto_scaling] : []
20+
content {
21+
min_workers = autoscale.value[0]
22+
max_workers = autoscale.value[1]
23+
}
24+
}
25+
26+
dynamic "aws_attributes" {
27+
for_each = var.aws_attributes == null ? [] : [var.aws_attributes]
28+
content {
29+
instance_profile_arn = var.add_instance_profile_to_workspace == true ? join("", databricks_instance_profile.shared.*.id) : lookup(aws_attributes.value, "instance_profile_arn", null)
30+
zone_id = lookup(aws_attributes.value, "zone_id", null)
31+
first_on_demand = lookup(aws_attributes.value, "first_on_demand", null)
32+
availability = lookup(aws_attributes.value, "availability", null)
33+
spot_bid_price_percent = lookup(aws_attributes.value, "spot_bid_price_percent ", null)
34+
ebs_volume_count = lookup(aws_attributes.value, "ebs_volume_count ", null)
35+
ebs_volume_size = lookup(aws_attributes.value, "ebs_volume_size ", null)
36+
}
37+
}
38+
39+
autotermination_minutes = var.cluster_autotermination_minutes
40+
custom_tags = var.custom_tags != null ? merge(var.custom_tags, local.shared_tags) : merge(local.shared_tags)
41+
42+
spark_conf = var.spark_conf
1443
}
1544

1645
notebook_task {
@@ -43,18 +72,51 @@ resource "databricks_job" "new_cluster_new_job_new_notebooks" {
4372
}
4473
}
4574
}
46-
4775
# ------------------------------------------------
48-
# 2. EXISTING CLUSTER WITH NEW NOTEBOOKS
76+
# 2. NEW CLUSTER WITH EXITING NOTEBOOKS
4977
# ------------------------------------------------
50-
resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
51-
for_each = (var.deploy_jobs == true && var.cluster_id != null && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
78+
resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
79+
for_each = (var.deploy_jobs == true && var.cluster_id == null && var.deploy_job_cluster == true && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
5280

53-
name = "${each.value.job_name} (Terraform managed)"
54-
existing_cluster_id = local.cluster_info
81+
name = "${each.value.job_name} (Terraform managed)"
82+
83+
new_cluster {
84+
policy_id = var.cluster_policy_id == null && var.deploy_cluster_policy == false ? null : local.cluster_policy_id
85+
spark_version = var.spark_version != null ? var.spark_version : data.databricks_spark_version.latest.id
86+
node_type_id = var.deploy_worker_instance_pool != true ? local.worker_node_type : null
87+
instance_pool_id = var.deploy_worker_instance_pool == true ? join("", databricks_instance_pool.worker_instance_nodes.*.id) : null
88+
driver_node_type_id = var.deploy_worker_instance_pool != true ? local.driver_node_type : null
89+
num_workers = var.fixed_value != null ? var.fixed_value : null
90+
91+
dynamic "autoscale" {
92+
for_each = var.auto_scaling != null ? [var.auto_scaling] : []
93+
content {
94+
min_workers = autoscale.value[0]
95+
max_workers = autoscale.value[1]
96+
}
97+
}
98+
99+
dynamic "aws_attributes" {
100+
for_each = var.aws_attributes == null ? [] : [var.aws_attributes]
101+
content {
102+
instance_profile_arn = var.add_instance_profile_to_workspace == true ? join("", databricks_instance_profile.shared.*.id) : lookup(aws_attributes.value, "instance_profile_arn", null)
103+
zone_id = lookup(aws_attributes.value, "zone_id", null)
104+
first_on_demand = lookup(aws_attributes.value, "first_on_demand", null)
105+
availability = lookup(aws_attributes.value, "availability", null)
106+
spot_bid_price_percent = lookup(aws_attributes.value, "spot_bid_price_percent ", null)
107+
ebs_volume_count = lookup(aws_attributes.value, "ebs_volume_count ", null)
108+
ebs_volume_size = lookup(aws_attributes.value, "ebs_volume_size ", null)
109+
}
110+
}
111+
112+
autotermination_minutes = var.cluster_autotermination_minutes
113+
custom_tags = var.custom_tags != null ? merge(var.custom_tags, local.shared_tags) : merge(local.shared_tags)
114+
115+
spark_conf = var.spark_conf
116+
}
55117

56118
notebook_task {
57-
notebook_path = lookup(each.value, "path", "${data.databricks_current_user.me.home}/${each.value.job_name}")
119+
notebook_path = lookup(each.value, "path")
58120
base_parameters = var.task_parameters
59121
}
60122

@@ -83,22 +145,18 @@ resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
83145
}
84146
}
85147
}
148+
86149
# ------------------------------------------------
87-
# 3. NEW CLUSTER WITH EXITING NOTEBOOKS
150+
# 3. EXISTING CLUSTER WITH NEW NOTEBOOKS
88151
# ------------------------------------------------
89-
resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
90-
for_each = (var.deploy_jobs == true && var.cluster_id == null && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
91-
92-
name = "${each.value.job_name} (Terraform managed)"
152+
resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
153+
for_each = (var.deploy_jobs == true && (var.cluster_id != null || var.deploy_cluster == true) && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
93154

94-
new_cluster {
95-
num_workers = var.num_workers
96-
spark_version = data.databricks_spark_version.latest.id
97-
node_type_id = join("", data.databricks_node_type.cluster_node_type.*.id)
98-
}
155+
name = "${each.value.job_name} (Terraform managed)"
156+
existing_cluster_id = local.cluster_info
99157

100158
notebook_task {
101-
notebook_path = lookup(each.value, "path")
159+
notebook_path = lookup(each.value, "path", "${data.databricks_current_user.me.home}/${each.value.job_name}")
102160
base_parameters = var.task_parameters
103161
}
104162

@@ -132,7 +190,7 @@ resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
132190
# 4. EXISTING CLUSTER WITH EXITING NOTEBOOKS
133191
# ------------------------------------------------
134192
resource "databricks_job" "existing_cluster_new_job_existing_notebooks" {
135-
for_each = (var.deploy_jobs == true && var.cluster_id != null && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
193+
for_each = var.deploy_jobs == true && (var.cluster_id != null || var.deploy_cluster == true) && var.remote_notebooks != null ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
136194

137195
name = "${each.value.job_name} (Terraform managed)"
138196
existing_cluster_id = local.cluster_info
@@ -166,4 +224,4 @@ resource "databricks_job" "existing_cluster_new_job_existing_notebooks" {
166224
pause_status = lookup(schedule.value, "pause_status", null)
167225
}
168226
}
169-
}
227+
}

‎locals.tf

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,14 @@ locals {
66
}
77
)
88

9-
# JOBS CLUSTER
10-
cluster_info = var.cluster_id == null ? join("", databricks_cluster.cluster.*.id) : var.cluster_id
9+
# CLUSTER
10+
11+
type_of_cluster = var.fixed_value == 0 ? join("", databricks_cluster.cluster.*.id) : join("", databricks_cluster.single_node_cluster.*.id)
12+
13+
cluster_info = var.cluster_id == null ? local.type_of_cluster : var.cluster_id
1114

1215
# USERS
13-
databricks_username = var.databricks_username != "" ? var.databricks_username : "${data.databricks_current_user.me.alphanumeric}"
16+
databricks_username = var.databricks_username != "" ? var.databricks_username : data.databricks_current_user.me.alphanumeric
1417
databricks_displayname = var.databricks_username != "" ? "${var.databricks_username} (Terraform managed)" : data.databricks_current_user.me.alphanumeric
1518

1619
# WORKER NODE TYPE

0 commit comments

Comments
 (0)