test

Author: tomarv2

cluster.tf

@@ -42,7 +42,9 @@ resource "databricks_cluster" "cluster" {
   }
 
   autotermination_minutes = var.cluster_autotermination_minutes
-  custom_tags             = merge(local.shared_tags)
+  #custom_tags             = merge(local.shared_tags)
+  custom_tags              = var.custom_tags != null ? merge(var.custom_tags, local.shared_tags) : merge(local.shared_tags)
+
   spark_conf              = var.spark_conf
 }
 

job.tf

@@ -3,12 +3,15 @@
 # 1. NEW CLUSTER WITH NEW NOTEBOOKS
 # ------------------------------------------------
 resource "databricks_job" "new_cluster_new_job_new_notebooks" {
-  for_each = (var.deploy_jobs == true && var.existing_cluster == false && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
-  #for_each = (var.deploy_jobs == true && var.cluster_id == null && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
+  for_each = (var.deploy_jobs == true && var.cluster_id == null && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
 
   name = "${each.value.job_name} (Terraform managed)"
 
-  existing_cluster_id = local.cluster_info
+  new_cluster {
+    num_workers   = var.num_workers
+    spark_version = data.databricks_spark_version.latest.id
+    node_type_id  = join("", data.databricks_node_type.cluster_node_type.*.id)
+  }
 
   notebook_task {
     notebook_path   = lookup(each.value, "path", "${data.databricks_current_user.me.home}/${each.value.job_name}")
@@ -40,20 +43,18 @@ resource "databricks_job" "new_cluster_new_job_new_notebooks" {
     }
   }
 }
+
 # ------------------------------------------------
-# 2. NEW CLUSTER WITH EXITING NOTEBOOKS
+# 2. EXISTING CLUSTER WITH NEW NOTEBOOKS
 # ------------------------------------------------
-resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
-  for_each = (var.deploy_jobs == true && var.existing_cluster == false && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
-  #for_each = (var.deploy_jobs == true && var.cluster_id == null && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
-
-
-  name = "${each.value.job_name} (Terraform managed)"
+resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
+  for_each = (var.deploy_jobs == true && var.cluster_id != null && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
 
+  name                = "${each.value.job_name} (Terraform managed)"
   existing_cluster_id = local.cluster_info
 
   notebook_task {
-    notebook_path   = lookup(each.value, "path")
+    notebook_path   = lookup(each.value, "path", "${data.databricks_current_user.me.home}/${each.value.job_name}")
     base_parameters = var.task_parameters
   }
 
@@ -82,18 +83,22 @@ resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
     }
   }
 }
-
 # ------------------------------------------------
-# 3. EXISTING CLUSTER WITH NEW NOTEBOOKS
+# 3. NEW CLUSTER WITH EXITING NOTEBOOKS
 # ------------------------------------------------
-resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
-  for_each = (var.deploy_jobs == true && var.cluster_id != null && var.local_notebooks != null) ? { for p in var.local_notebooks : "${p.job_name}-${p.local_path}" => p } : {}
+resource "databricks_job" "new_cluster_new_job_existing_notebooks" {
+  for_each = (var.deploy_jobs == true && var.cluster_id == null && var.remote_notebooks != null) ? { for p in var.remote_notebooks : "${p.job_name}-${p.path}" => p } : {}
 
-  name                = "${each.value.job_name} (Terraform managed)"
-  existing_cluster_id = local.cluster_info
+  name = "${each.value.job_name} (Terraform managed)"
+
+  new_cluster {
+    num_workers   = var.num_workers
+    spark_version = data.databricks_spark_version.latest.id
+    node_type_id  = join("", data.databricks_node_type.cluster_node_type.*.id)
+  }
 
   notebook_task {
-    notebook_path   = lookup(each.value, "path", "${data.databricks_current_user.me.home}/${each.value.job_name}")
+    notebook_path   = lookup(each.value, "path")
     base_parameters = var.task_parameters
   }
 
@@ -122,6 +127,7 @@ resource "databricks_job" "existing_cluster_new_job_new_notebooks" {
     }
   }
 }
+
 # ------------------------------------------------
 # 4. EXISTING CLUSTER WITH EXITING NOTEBOOKS
 # ------------------------------------------------
@@ -160,4 +166,4 @@ resource "databricks_job" "existing_cluster_new_job_existing_notebooks" {
       pause_status           = lookup(schedule.value, "pause_status", null)
     }
   }
-}
+}

variables.tf

@@ -402,3 +402,9 @@ variable "cluster_name" {
   description = "Cluster name"
   default     = null
 }
+
+variable "custom_tags" {
+  type        = any
+  description = "Extra custom tags"
+  default     = null
+}