Feature/upgrade data hub to 5.7

README.md

@@ -47,7 +47,7 @@ The HSK was built and tested with the following prerequisites:
 - [MarkLogic Data Hub Central v5.5.1](http://developer.marklogic.com/download/binaries/dhf/marklogic-data-hub-central-5.5.1.war)
 - MarkLogic Server >= v10.0-7
 
-### Installation Steps:
+### Installation Steps
 
 > Note: Installation steps assume a MarkLogic Server user/role with sufficient privileges is specified. Refer to the MarkLogic Data Hub documentation if needed.
 

build.gradle

@@ -1,7 +1,7 @@
 buildscript {
   repositories {
     mavenLocal()
-    maven {url 'http://developer.marklogic.com/maven2/'}
+    maven {url 'https://developer.marklogic.com/maven2/'}
     jcenter()
   }
 
@@ -20,7 +20,7 @@ plugins {
   id 'net.saliman.properties' version '1.4.6'
 
   // Data Hub plugin
-  id 'com.marklogic.ml-data-hub' version '5.5.1'
+  id 'com.marklogic.ml-data-hub' version '5.7.0'
 
   //Data services needs these
   id 'com.marklogic.ml-development-tools' version '5.4.0'
@@ -30,7 +30,7 @@ repositories {
   maven {url "https://plugins.gradle.org/m2/"}
   mavenCentral()
   jcenter()
-  maven {url 'http://developer.marklogic.com/maven2/'}
+  maven {url 'https://developer.marklogic.com/maven2/'}
   if (project.hasProperty("testing")) {
     mavenLocal()
   }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-bin.zip
-zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME

src/main/hub-internal-config/security/amps/deleteProvenance.json

@@ -0,0 +1,6 @@
+{
+  "local-name" : "deleteProvenance",
+  "document-uri" : "/data-hub/5/provenance/prov-lib.sjs",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "ps-internal" ]
+}

src/main/hub-internal-config/security/amps/migrate-provenance-record.json

@@ -0,0 +1,7 @@
+{
+  "namespace" : "http://marklogic.com/datahub/migrate-prov",
+  "local-name" : "migrate-provenance",
+  "document-uri" : "/data-hub/5/provenance/migrate-provenance.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "ps-internal" ]
+}

src/main/hub-internal-config/security/amps/update-provenance-record.json

@@ -0,0 +1,7 @@
+{
+  "namespace" : "http://marklogic.com/data-hub/dh-provenance-services",
+  "local-name" : "dhps-document-update",
+  "document-uri" : "/data-hub/5/provenance/dh-provenance.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "ps-internal" ]
+}

src/main/hub-internal-config/security/amps/write-provenance-record.json

@@ -0,0 +1,7 @@
+{
+  "namespace" : "http://marklogic.com/data-hub/dh-provenance-services",
+  "local-name" : "dhps-document-insert",
+  "document-uri" : "/data-hub/5/provenance/dh-provenance.xqy",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "ps-internal" ]
+}

src/main/hub-internal-config/security/privileges/data-hub-delete-provenance.json

@@ -0,0 +1,5 @@
+{
+  "privilege-name": "data-hub-delete-provenance",
+  "action": "http://marklogic.com/data-hub/privileges/delete-provenance",
+  "kind": "execute"
+}

src/main/hub-internal-config/security/roles/data-hub-admin.json

@@ -1,6 +1,6 @@
 {
   "role-name": "data-hub-admin",
-  "description": "5.5.1: Permits actions pertaining to administering a Data Hub application",
+  "description": "5.7.0: Permits actions pertaining to administering a Data Hub application",
   "role": [
     "data-hub-developer"
   ],

src/main/hub-internal-config/security/roles/data-hub-common.json

@@ -111,6 +111,11 @@
       "privilege-name": "xdmp:xslt-invoke",
       "action": "http://marklogic.com/xdmp/privileges/xslt-invoke",
       "kind": "execute"
+    },
+    {
+      "privilege-name": "xdmp:xslt-eval",
+      "action": "http://marklogic.com/xdmp/privileges/xslt-eval",
+      "kind": "execute"
     }
   ]
 }

src/main/hub-internal-config/security/roles/data-hub-operator.json

@@ -83,8 +83,8 @@
       "kind": "execute"
     },
     {
-      "privilege-name": "data-hub-prune-provenance",
-      "action": "http://marklogic.com/data-hub/privileges/prune-provenance",
+      "privilege-name": "data-hub-delete-provenance",
+      "action": "http://marklogic.com/data-hub/privileges/delete-provenance",
       "kind": "execute"
     }
   ]

src/main/hub-internal-config/security/roles/hub-central-entity-model-reader.json

@@ -3,6 +3,7 @@
   "description": "Permits reading entity models in Hub Central",
   "role": [
     "hub-central-user",
+    "data-hub-common-writer",
     "data-hub-entity-model-reader"
   ]
 }

src/main/hub-internal-config/security/roles/hub-central-mapping-reader.json

@@ -4,7 +4,8 @@
   "role": [
     "hub-central-user",
     "data-hub-entity-model-reader",
-    "data-hub-mapping-reader"
+    "data-hub-mapping-reader",
+    "data-hub-step-definition-reader"
   ],
   "privilege": [
     {

src/main/hub-internal-config/triggers/ml-dh-draft-entity-validate-create.json

@@ -0,0 +1,39 @@
+{
+  "name": "ml-dh-draft-entity-validate-create",
+  "description": "MarkLogic Data Hub draft entity model creation validation pre-commit trigger",
+  "event": {
+    "data-event": {
+      "collection-scope": {
+        "uri": "http://marklogic.com/entity-services/models/draft"
+      },
+      "document-content": {
+        "update-kind": "create"
+      },
+      "when": "pre-commit"
+    }
+  },
+  "module": "data-hub/4/triggers/entity-model-validate-trigger.xqy",
+  "module-db": "%%mlModulesDbName%%",
+  "module-root": "/",
+  "enabled": true,
+  "recursive": true,
+  "task-priority": "normal",
+  "permission": [
+    {
+      "role-name": "%%mlFlowDeveloperRole%%",
+      "capability": "update"
+    },
+    {
+      "role-name": "%%mlFlowOperatorRole%%",
+      "capability": "read"
+    },
+    {
+      "role-name": "data-hub-developer",
+      "capability": "update"
+    },
+    {
+      "role-name": "data-hub-common",
+      "capability": "read"
+    }
+  ]
+}

src/main/hub-internal-config/triggers/ml-dh-draft-entity-validate-modify.json

@@ -0,0 +1,39 @@
+{
+  "name": "ml-dh-draft-entity-validate-modify",
+  "description": "MarkLogic Data Hub draft entity model update validation pre-commit trigger",
+  "event": {
+    "data-event": {
+      "collection-scope": {
+        "uri": "http://marklogic.com/entity-services/models/draft"
+      },
+      "document-content": {
+        "update-kind": "modify"
+      },
+      "when": "pre-commit"
+    }
+  },
+  "module": "data-hub/4/triggers/entity-model-validate-trigger.xqy",
+  "module-db": "%%mlModulesDbName%%",
+  "module-root": "/",
+  "enabled": true,
+  "recursive": true,
+  "task-priority": "normal",
+  "permission": [
+    {
+      "role-name": "%%mlFlowDeveloperRole%%",
+      "capability": "update"
+    },
+    {
+      "role-name": "%%mlFlowOperatorRole%%",
+      "capability": "read"
+    },
+    {
+      "role-name": "data-hub-developer",
+      "capability": "update"
+    },
+    {
+      "role-name": "data-hub-common",
+      "capability": "read"
+    }
+  ]
+}

src/main/hub-internal-config/triggers/ml-dh-entity-validate-create.json

@@ -17,7 +17,7 @@
   "module-root": "/",
   "enabled": true,
   "recursive": true,
-  "task-priority": "normal",
+  "task-priority": "higher",
   "permission": [
     {
       "role-name": "%%mlFlowDeveloperRole%%",

src/main/hub-internal-config/triggers/ml-dh-entity-validate-modify.json

@@ -17,7 +17,7 @@
   "module-root": "/",
   "enabled": true,
   "recursive": true,
-  "task-priority": "normal",
+  "task-priority": "higher",
   "permission": [
     {
       "role-name": "%%mlFlowDeveloperRole%%",

src/main/ml-config/database-fields/final-database.xml

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <database-properties xmlns="http://marklogic.com/manage">
   <database-name>%%mlFinalDbName%%</database-name>
   <path-namespaces>
@@ -9,32 +10,32 @@
   <fields>
     <field>
       <field-name>datahubCreatedOn</field-name>
-      <metadata/>
+      <metadata />
       <stemmed-searches>off</stemmed-searches>
       <word-searches>false</word-searches>
       <field-value-searches>true</field-value-searches>
       <field-value-positions>false</field-value-positions>
       <fast-phrase-searches>false</fast-phrase-searches>
       <fast-case-sensitive-searches>false</fast-case-sensitive-searches>
       <fast-diacritic-sensitive-searches>false</fast-diacritic-sensitive-searches>
-      <word-lexicons/>
-      <included-elements/>
-      <excluded-elements/>
-      <tokenizer-overrides/>
+      <word-lexicons />
+      <included-elements />
+      <excluded-elements />
+      <tokenizer-overrides />
     </field>
     <field>
       <field-name>datahubCreatedByJob</field-name>
-      <metadata/>
+      <metadata />
       <stemmed-searches>off</stemmed-searches>
       <word-searches>true</word-searches>
       <field-value-searches>true</field-value-searches>
       <field-value-positions>false</field-value-positions>
       <fast-phrase-searches>false</fast-phrase-searches>
       <fast-case-sensitive-searches>false</fast-case-sensitive-searches>
       <fast-diacritic-sensitive-searches>false</fast-diacritic-sensitive-searches>
-      <word-lexicons/>
-      <included-elements/>
-      <excluded-elements/>
+      <word-lexicons />
+      <included-elements />
+      <excluded-elements />
       <tokenizer-overrides>
         <tokenizer-override>
           <character>-</character>
@@ -44,17 +45,17 @@
     </field>
     <field>
       <field-name>datahubCreatedByStep</field-name>
-      <metadata/>
+      <metadata />
       <stemmed-searches>off</stemmed-searches>
       <word-searches>true</word-searches>
       <field-value-searches>true</field-value-searches>
       <field-value-positions>false</field-value-positions>
       <fast-phrase-searches>false</fast-phrase-searches>
       <fast-case-sensitive-searches>false</fast-case-sensitive-searches>
       <fast-diacritic-sensitive-searches>false</fast-diacritic-sensitive-searches>
-      <word-lexicons/>
-      <included-elements/>
-      <excluded-elements/>
+      <word-lexicons />
+      <included-elements />
+      <excluded-elements />
       <tokenizer-overrides>
         <tokenizer-override>
           <character>_</character>
@@ -68,18 +69,18 @@
     </field>
     <field>
       <field-name>datahubCreatedInFlow</field-name>
-      <metadata/>
+      <metadata />
       <stemmed-searches>off</stemmed-searches>
       <word-searches>true</word-searches>
       <field-value-searches>true</field-value-searches>
       <field-value-positions>false</field-value-positions>
       <fast-phrase-searches>false</fast-phrase-searches>
       <fast-case-sensitive-searches>false</fast-case-sensitive-searches>
       <fast-diacritic-sensitive-searches>false</fast-diacritic-sensitive-searches>
-      <word-lexicons/>
-      <included-elements/>
-      <excluded-elements/>
-      <tokenizer-overrides/>
+      <word-lexicons />
+      <included-elements />
+      <excluded-elements />
+      <tokenizer-overrides />
     </field>
     <field>
       <field-name>datahubSourceName</field-name>
@@ -94,10 +95,10 @@
       <fast-phrase-searches>false</fast-phrase-searches>
       <fast-case-sensitive-searches>false</fast-case-sensitive-searches>
       <fast-diacritic-sensitive-searches>false</fast-diacritic-sensitive-searches>
-      <word-lexicons/>
-      <included-elements/>
-      <excluded-elements/>
-      <tokenizer-overrides/>
+      <word-lexicons />
+      <included-elements />
+      <excluded-elements />
+      <tokenizer-overrides />
     </field>
     <field>
       <field-name>datahubSourceType</field-name>
@@ -112,16 +113,16 @@
       <fast-phrase-searches>false</fast-phrase-searches>
       <fast-case-sensitive-searches>false</fast-case-sensitive-searches>
       <fast-diacritic-sensitive-searches>false</fast-diacritic-sensitive-searches>
-      <word-lexicons/>
-      <included-elements/>
-      <excluded-elements/>
-      <tokenizer-overrides/>
+      <word-lexicons />
+      <included-elements />
+      <excluded-elements />
+      <tokenizer-overrides />
     </field>
   </fields>
   <range-field-indexes>
     <range-field-index>
       <scalar-type>dateTime</scalar-type>
-      <collation/>
+      <collation />
       <field-name>datahubCreatedOn</field-name>
       <range-value-positions>false</range-value-positions>
       <invalid-values>reject</invalid-values>