Initial commit

Author: kevchant

.github/workflows/Single-Serverless-Pool.yml

@@ -0,0 +1,43 @@
+name: Update Azure Synapse Serverless SQL Pool
+
+# Sample of how to deploy to an Azure Synapse Serverless SQL Pool using GitHub Actions
+# You can use a self-hosted runner installed locally for this to work
+# You can also use a GitHub-hosted runner
+
+# For this pipeline you also need to setup the below encrypted secrets (https://bit.ly/3nd0Jj8)
+  # sqlinstance - Your Serverless SQL Pool endpoint 
+    #(the one that ends with -ondemand)
+  # database - The database in the SQL Pool you want the update deployed to
+    # Note it has to exist BEFORE this the scripts are run
+  # UserName - User name to connect to the Serverless SQL Pool endpoint
+    # try and keep this secret
+  # Pw - Password of above user, definitely keep this one secret
+
+#Sets the trigger to update when update is pushed to main branch
+on:
+  push:
+    branches: 
+      - main
+
+jobs:
+
+  # Job to install the scripts
+  ServerlessPool1:
+    # Easier to use Github-hosted runner if updating in GitHub
+    runs-on: windows-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      # Note that I am using latest version of action
+      - uses: actions/checkout@v2.4.0 
+
+      # install dbops PowerShell module
+      - name: Install dbops module
+        run: 'Install-Module -Name dbops -Force -PassThru'
+        
+      # Run migration-based scripts
+      - name: Run migration-based scripts
+        run: |
+          $SecurePw=ConvertTo-SecureString ${{ secrets.Pw }} –asplaintext –force
+          Install-DBOScript -ScriptPath scripts -sqlinstance ${{ secrets.sqlinstance }} -Database ${{ secrets.database }} -UserName ${{ secrets.UserName }} -Password $SecurePw -SchemaVersionTable $null

README.md

@@ -0,0 +1,13 @@
+# Azure Synapse Serverless SQL Pool Database Project using GitHub Actions
+
+Example of a Migration-Based deployment that deploys to an Azure Synapse serverless SQL Pool using GitHub Actions.
+
+It uses a yaml pipeline, which is in the '/.github/workflows' folder.
+
+In order to use it in GitHub Actions you can either import or fork this repository into another GitHub repository.
+
+Afterwards, you can select the yaml file in the GitHub repository and tailor the pipeline to suit your needs. Or, you can clone your repository locally and change the pure yaml file there.
+
+Please note that the databases have to exist in the serverless SQL Pools for this to work. In addition, you might want to create a file in Azure Data Lake storage that contains the heading used for the SchemaVersions table (https://dbup.readthedocs.io/en/latest/more-info/journaling/). From there, you can try using it as the SchemaVersions table in the code.
+
+This repository is provided "as is" based on the MIT license (https://opensource.org/licenses/MIT). Basically, I am not responsible for your use of it.

scripts/Script0001 - Create table.sql

@@ -0,0 +1,43 @@
+IF NOT EXISTS (SELECT * FROM sys.external_file_formats WHERE name = 'SynapseParquetFormat')
+	CREATE EXTERNAL FILE FORMAT [SynapseParquetFormat]
+	WITH ( FORMAT_TYPE = PARQUET)
+GO
+
+IF NOT EXISTS (SELECT * FROM sys.external_data_sources WHERE name = 'nyctlc_azureopendatastorage_blob_core_windows_net')
+	CREATE EXTERNAL DATA SOURCE [nyctlc_azureopendatastorage_blob_core_windows_net]
+	WITH (
+		LOCATION   = 'https://azureopendatastorage.blob.core.windows.net/nyctlc',
+	)
+Go
+
+IF NOT EXISTS (SELECT * FROM sys.external_tables WHERE name = 'nyc_tlc_yellow_trip_ext')
+CREATE EXTERNAL TABLE nyc_tlc_yellow_trip_ext (
+	[vendorID] varchar(8000),
+	[tpepPickupDateTime] datetime2(7),
+	[tpepDropoffDateTime] datetime2(7),
+	[passengerCount] int,
+	[tripDistance] float,
+	[puLocationId] varchar(8000),
+	[doLocationId] varchar(8000),
+	[startLon] float,
+	[startLat] float,
+	[endLon] float,
+	[endLat] float,
+	[rateCodeId] int,
+	[storeAndFwdFlag] varchar(8000),
+	[paymentType] varchar(8000),
+	[fareAmount] float,
+	[extra] float,
+	[mtaTax] float,
+	[improvementSurcharge] varchar(8000),
+	[tipAmount] float,
+	[tollsAmount] float,
+	[totalAmount] float
+	)
+	WITH (
+    LOCATION = 'yellow/puYear=2014/puMonth=3/*.parquet',
+	-- LOCATION = 'yellow/puYear=*/puMonth=*/*.parquet'
+	DATA_SOURCE = [nyctlc_azureopendatastorage_blob_core_windows_net],
+	FILE_FORMAT = [SynapseParquetFormat]
+	)
+GO

scripts/Script0002 - Alter table.sql

@@ -0,0 +1,34 @@
+IF EXISTS (SELECT * FROM sys.external_tables WHERE name = 'nyc_tlc_yellow_trip_ext')
+DROP EXTERNAL TABLE nyc_tlc_yellow_trip_ext
+GO
+
+CREATE EXTERNAL TABLE nyc_tlc_yellow_trip_ext (
+	[vendorID] varchar(8000),
+	[tpepPickupDateTime] datetime2(7),
+	[tpepDropoffDateTime] datetime2(7),
+	[passengerCount] int,
+	[tripDistance] float,
+	[puLocationId] varchar(8000),
+	[doLocationId] varchar(8000),
+	[startLon] float,
+	[startLat] float,
+	[endLon] float,
+	[endLat] float,
+	[rateCodeId] int,
+	[storeAndFwdFlag] varchar(8000),
+	[paymentType] varchar(8000),
+	[fareAmount] float,
+	[extra] float,
+	[mtaTax] float,
+	[improvementSurcharge] varchar(8000),
+	[tipAmount] float,
+	[tollsAmount] float,
+	[totalAmount] float
+	)
+	WITH (
+    LOCATION = 'yellow/puYear=2014/puMonth=3/*.parquet',
+	-- LOCATION = 'yellow/puYear=*/puMonth=*/*.parquet'
+	DATA_SOURCE = [nyctlc_azureopendatastorage_blob_core_windows_net],
+	FILE_FORMAT = [SynapseParquetFormat]
+	)
+GO