From d1c841b47386b5e7e5cb09d2d34ef4bebf7448c4 Mon Sep 17 00:00:00 2001 From: Paul Andrew <60190428+mrpaulandrew@users.noreply.github.com> Date: Sat, 26 Feb 2022 10:40:27 +0000 Subject: [PATCH] ARM template and parameters deployed on 2-26-2022 10:40:26, based on the collaboration branch's commit ID: 8d07f3c5d30e1d16f8bdfeccca7bcb39d5eb76a0 --- TrainingFactoryDev/ARMTemplateForFactory.json | 3264 +++++++++++++++++ .../ARMTemplateParametersForFactory.json | 74 + .../TrainingFactoryDev_GlobalParameters.json | 10 + .../ArmTemplateParameters_master.json | 80 + .../linkedTemplates/ArmTemplate_0.json | 1013 +++++ .../linkedTemplates/ArmTemplate_1.json | 954 +++++ .../linkedTemplates/ArmTemplate_2.json | 1270 +++++++ .../linkedTemplates/ArmTemplate_master.json | 231 ++ 8 files changed, 6896 insertions(+) create mode 100644 TrainingFactoryDev/ARMTemplateForFactory.json create mode 100644 TrainingFactoryDev/ARMTemplateParametersForFactory.json create mode 100644 TrainingFactoryDev/globalParameters/TrainingFactoryDev_GlobalParameters.json create mode 100644 TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json create mode 100644 TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json create mode 100644 TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json create mode 100644 TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json create mode 100644 TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json diff --git a/TrainingFactoryDev/ARMTemplateForFactory.json b/TrainingFactoryDev/ARMTemplateForFactory.json new file mode 100644 index 0000000..786580a --- /dev/null +++ b/TrainingFactoryDev/ARMTemplateForFactory.json @@ -0,0 +1,3264 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name", + "defaultValue": "TrainingFactoryDev" + }, + "TrainingStore01_connectionString": { + "type": "secureString", + "metadata": "Secure string for 'connectionString' of 'TrainingStore01'" + }, + "traininglak01_accountKey": { + "type": "secureString", + "metadata": "Secure string for 'accountKey' of 'traininglak01'" + }, + "traininglake01noneKV_accountKey": { + "type": "secureString", + "metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'" + }, + "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "@linkedService().DBConnectionSecret" + }, + "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "@linkedService().KeyVaultSecretName" + }, + "BatchForTraining01_properties_typeProperties_batchUri": { + "type": "string", + "defaultValue": "https://batchfortraining01.uksouth.batch.azure.com" + }, + "BatchForTraining01_properties_typeProperties_poolName": { + "type": "string", + "defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654" + }, + "BatchForTraining01_properties_typeProperties_accountName": { + "type": "string", + "defaultValue": "batchfortraining01" + }, + "GenericKeys_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "@{linkedService().baseUrl}" + }, + "LaptopFiles_properties_typeProperties_host": { + "type": "string", + "defaultValue": "C:\\ADFRoot\\" + }, + "LaptopFiles_properties_typeProperties_userId": { + "type": "string", + "defaultValue": "ADFIRUser" + }, + "TrainingKeys01_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "https://TrainingKeys01.vault.azure.net/" + }, + "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "https://TrainingKeys01.vault.azure.net/" + }, + "trainingdb01_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "traininglak01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01noneKV_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { + "type": "string", + "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" + }, + "AzureKeyVault1_properties_privateLinkResourceId": { + "type": "string", + "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" + }, + "AzureKeyVault1_properties_groupId": { + "type": "string", + "defaultValue": "vault" + }, + "AzureKeyVault1_properties_fqdns": { + "type": "array", + "defaultValue": [ + "trainingkeys01.vaultcore.azure.net" + ] + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Upload Person", + "description": "hfdhgfdhfd", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopFilePersonCSV", + "type": "DatasetReference", + "parameters": {} + } + ], + "outputs": [ + { + "referenceName": "LakeFilePersonCSV", + "type": "DatasetReference", + "parameters": {} + } + ] + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-29T13:50:28Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]", + "[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Upload Any File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": "Person.csv" + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": "Person.csv" + } + } + ] + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:46Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", + "[concat(variables('factoryId'), '/datasets/LakeFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "GetMetadata", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "LaptopFolders", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People" + } + }, + "fieldList": [ + "childItems" + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + } + }, + { + "name": "Upload Files", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.childItems", + "type": "Expression" + }, + "activities": [ + { + "name": "Upload File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:45Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LaptopFolders')]", + "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", + "[concat(variables('factoryId'), '/datasets/LakeFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "UploadFiles", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Log Upload Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy By File Type", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@substring(item().FileName,add(indexof(item().FileName,'.'),1),sub(length(item().FileName),add(indexof(item().FileName,'.'),1)))", + "type": "Expression" + }, + "cases": [ + { + "value": "csv", + "activities": [ + { + "name": "CSV Copy", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "additionalColumns": [ + { + "name": "FilePath", + "value": "$$FILEPATH" + }, + { + "name": "Environment", + "value": { + "value": "@pipeline().globalParameters.Environment", + "type": "Expression" + } + }, + { + "name": "PipelineName", + "value": { + "value": "@pipeline().Pipeline", + "type": "Expression" + } + }, + { + "name": "RunId", + "value": { + "value": "@pipeline().RunId", + "type": "Expression" + } + }, + { + "name": "RunDate", + "value": { + "value": "@utcnow()", + "type": "Expression" + } + } + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": false, + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + }, + { + "value": "parquet", + "activities": [ + { + "name": "Parquet Copy", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": false, + "enablePartitionDiscovery": false + } + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "ParquetWriteSettings" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopFilesParquet", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFileParquet", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + } + ], + "defaultActivities": [ + { + "name": "Binary Copy", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": false + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "LaptopFilesBinary", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFilesBinary", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ] + } + }, + { + "name": "Log Upload Start", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[LogUploadStart]", + "storedProcedureParameters": { + "FileCount": { + "type": "Int32", + "value": { + "value": "@activity('Get File List').output.count", + "type": "Expression" + } + }, + "TriggerId": { + "type": "String", + "value": { + "value": "@pipeline().TriggerId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Log Upload Finish", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "UploadFiles", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[LogUploadFinish]", + "storedProcedureParameters": { + "LogId": { + "value": { + "value": "@activity('Log Upload Start').output.firstRow.LogId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "variables": { + "DemoVariable": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:14Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", + "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]", + "[concat(variables('factoryId'), '/datasets/LaptopFilesBinary')]", + "[concat(variables('factoryId'), '/datasets/LakeFilesBinary')]", + "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", + "[concat(variables('factoryId'), '/datasets/LakeFiles')]", + "[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]", + "[concat(variables('factoryId'), '/datasets/LakeFileParquet')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/05 - Get File List Utility')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.GetFileQuery", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "GetFileQuery": { + "type": "string" + } + }, + "variables": { + "DemoVariable": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:14Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/06 - Example Global Param')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "06 - Get Any File List", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "05 - Get File List Utility", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "GetFileQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;" + } + } + }, + { + "name": "Set Pipe Output", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "06 - Get Any File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "WorkerOutput", + "value": { + "value": "@activity('06 - Get Any File List').output.pipelineRunId", + "type": "Expression" + } + } + }, + { + "name": "Set Global Param", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "06 - Get Any File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "GlobalParam", + "value": { + "value": "@concat(pipeline().globalParameters.Environment,'Test')", + "type": "Expression" + } + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "variables": { + "WorkerOutput": { + "type": "String" + }, + "GlobalParam": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/05 - Get File List Utility')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/1950 Activities')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "50", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Wait 1 Minute 1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 2", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 3", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 4", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 5", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 6", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 7", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 8", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 9", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 10", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 11", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 12", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 13", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 14", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 15", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 16", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 17", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 18", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 19", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 20", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 21", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 22", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 23", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 24", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 25", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 26", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 27", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 28", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 29", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 30", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 31", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 32", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 33", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 34", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 35", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 36", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 37", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 38", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 39", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [], + "lastPublishTime": "2022-02-03T14:39:03Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Custom Activity')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "HelloWorld", + "type": "Custom", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "command": "CustomActivity.exe", + "resourceLinkedService": { + "referenceName": "TrainingStore01", + "type": "LinkedServiceReference" + }, + "folderPath": "customactivities", + "referenceObjects": { + "linkedServices": [], + "datasets": [] + } + }, + "linkedServiceName": { + "referenceName": "BatchForTraining01", + "type": "LinkedServiceReference" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [], + "lastPublishTime": "2022-02-03T14:39:04Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]", + "[concat(variables('factoryId'), '/linkedServices/BatchForTraining01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Secret", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Set Key URL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": { + "value": "@variables('CompleteSecretURL')", + "type": "Expression" + }, + "method": "GET", + "headers": {}, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set Key URL", + "type": "SetVariable", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "variableName": "CompleteSecretURL", + "value": { + "value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')", + "type": "Expression" + } + } + }, + { + "name": "Nested Key Vault URL", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get Secret", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTableAnyKeyVault", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": "ConnectionString-trainingdb01", + "SchemaName": "sys", + "TableName": "objects", + "KeyVaultURL": { + "value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')", + "type": "Expression" + } + } + } + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "SecretName": { + "type": "string", + "defaultValue": "DemoKeyGetWithWebActivity" + } + }, + "variables": { + "CompleteSecretURL": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/AnyDatabaseTableAnyKeyVault')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Lazy Replication')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Table List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TableLookupQuery", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": "sys", + "TableName": "objects" + } + }, + "firstRowOnly": false + } + }, + { + "name": "Copy Tables", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Table List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Table List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy Table", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.TargetConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "SourceConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "TargetConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb02" + }, + "TableLookupQuery": { + "type": "string", + "defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'" + } + }, + "folder": { + "name": "Demo Pipelines/Dynamic Linked Services" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:43:42Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/AnyDatabaseTable')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Order Summary with Mapping')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Mapping Order Aggregation", + "type": "ExecuteDataFlow", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataflow": { + "referenceName": "MappingOrderAggregation", + "type": "DataFlowReference", + "parameters": {}, + "datasetParameters": { + "OrderHeader": {}, + "OrderLineDetails": {}, + "OrderSummary": {} + } + }, + "staging": {}, + "compute": { + "coreCount": 8, + "computeType": "General" + }, + "traceLevel": "Fine" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Flows" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:15Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/dataflows/MappingOrderAggregation')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Scale Out Level 1')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Bucket Ids", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Reset And Log Start", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Reset Bucket Process Map", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[GetBuckets]", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Call Buckets", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Ids", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Ids').output.value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 50, + "activities": [ + { + "name": "Call Level 2", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "Scale Out Level 2", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "BucketId": { + "value": "@{item().BucketId}", + "type": "Expression" + } + } + } + } + ] + } + }, + { + "name": "Log End", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Call Buckets", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Reset And Log Start", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[SetBucketLogEntry]", + "storedProcedureParameters": { + "LogId": { + "value": { + "value": "@activity('Reset And Log Start').output.firstRow.LogId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + }, + { + "name": "Reset And Log Start", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[SetBucketLogEntry]", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Reset Bucket Process Map", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[SetBucketProcesses]" + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", + "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]", + "[concat(variables('factoryId'), '/pipelines/Scale Out Level 2')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Bucket Contents", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]", + "storedProcedureParameters": { + "BucketId": { + "type": "Int32", + "value": { + "value": "@pipeline().parameters.BucketId", + "type": "Expression" + } + } + }, + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Execute Processes", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Contents", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Contents').output.value", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Run Process", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": { + "value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "BucketId": { + "type": "int" + } + }, + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", + "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 10')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Wait 10", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 10 + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnection", + "type": "LinkedServiceReference", + "parameters": { + "DBConnectionSecret": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnectionFromAnyKeyVault", + "type": "LinkedServiceReference", + "parameters": { + "KeyVaultURL": { + "value": "@dataset().KeyVaultURL", + "type": "Expression" + }, + "KeyVaultSecretName": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + }, + "KeyVaultURL": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": {} + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01noneKV", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "OrderDetailLines.parquet", + "folderPath": "Raw", + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [ + { + "name": "SalesOrderID", + "type": "INT32" + }, + { + "name": "SalesOrderDetailID", + "type": "INT32" + }, + { + "name": "OrderQty", + "type": "INT32" + }, + { + "name": "ProductID", + "type": "INT32" + }, + { + "name": "UnitPrice", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "UnitPriceDiscount", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "LineTotal", + "type": "DECIMAL", + "precision": 38, + "scale": 6 + }, + { + "name": "rowguid", + "type": "UTF8" + }, + { + "name": "ModifiedDate", + "type": "INT96" + } + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileOrderHeaderParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01noneKV", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "OrderHeader.parquet", + "folderPath": "Raw", + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [ + { + "name": "SalesOrderID", + "type": "INT32" + }, + { + "name": "RevisionNumber", + "type": "INT32" + }, + { + "name": "OrderDate", + "type": "INT96" + }, + { + "name": "DueDate", + "type": "INT96" + }, + { + "name": "ShipDate", + "type": "INT96" + }, + { + "name": "Status", + "type": "INT32" + }, + { + "name": "OnlineOrderFlag", + "type": "BOOLEAN" + }, + { + "name": "SalesOrderNumber", + "type": "UTF8" + }, + { + "name": "PurchaseOrderNumber", + "type": "UTF8" + }, + { + "name": "AccountNumber", + "type": "UTF8" + }, + { + "name": "CustomerID", + "type": "INT32" + }, + { + "name": "ShipToAddressID", + "type": "INT32" + }, + { + "name": "BillToAddressID", + "type": "INT32" + }, + { + "name": "ShipMethod", + "type": "UTF8" + }, + { + "name": "CreditCardApprovalCode", + "type": "UTF8" + }, + { + "name": "SubTotal", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "TaxAmt", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "Freight", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "TotalDue", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "Comment", + "type": "UTF8" + }, + { + "name": "rowguid", + "type": "UTF8" + }, + { + "name": "ModifiedDate", + "type": "INT96" + } + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "Person.csv", + "folderPath": "Landing", + "fileSystem": "datawarehouse" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [ + { + "type": "String" + }, + { + "type": "String" + }, + { + "type": "String" + } + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFiles')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": "Person.csv", + "folderPath": "ForUpload/People" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [ + { + "type": "String" + }, + { + "type": "String" + }, + { + "type": "String" + } + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFolders')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopsFiles')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/TableOrderSummary')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [ + { + "name": "SalesOrderNumber", + "type": "varchar" + }, + { + "name": "RecordCount", + "type": "int", + "precision": 10 + } + ], + "typeProperties": { + "schema": "dbo", + "table": "OrderSummary" + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "parameters": { + "DBConnectionSecret": { + "type": "string" + } + }, + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": { + "value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]", + "type": "Expression" + } + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "parameters": { + "KeyVaultURL": { + "type": "string" + }, + "KeyVaultSecretName": { + "type": "string" + } + }, + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "GenericKeys", + "type": "LinkedServiceReference", + "parameters": { + "baseUrl": { + "value": "@linkedService().KeyVaultURL", + "type": "Expression" + } + } + }, + "secretName": { + "value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]", + "type": "Expression" + } + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/GenericKeys')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/BatchForTraining01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBatch", + "typeProperties": { + "batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]", + "poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]", + "accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]", + "linkedServiceName": { + "referenceName": "TrainingStore01", + "type": "LinkedServiceReference" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/GenericKeys')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "description": "https://TrainingKeys01.vault.azure.net", + "parameters": { + "baseUrl": { + "type": "String" + } + }, + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFiles')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "FileServer", + "typeProperties": { + "host": "[parameters('LaptopFiles_properties_typeProperties_host')]", + "userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]", + "password": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "ADFIRUser" + } + }, + "connectVia": { + "referenceName": "PaulsLaptopIR", + "type": "IntegrationRuntimeReference" + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]", + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingKeys01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]", + "credential": { + "referenceName": "TrainingCredential01", + "type": "CredentialReference" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/credentials/TrainingCredential01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingStore01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobStorage", + "typeProperties": { + "connectionString": "[parameters('TrainingStore01_connectionString')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/trainingdb01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/traininglak01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglak01_properties_typeProperties_url')]", + "accountKey": { + "type": "SecureString", + "value": "[parameters('traininglak01_accountKey')]" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/traininglake01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglake01_properties_typeProperties_url')]", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "traininglake01" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/traininglake01noneKV')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglake01noneKV_properties_typeProperties_url')]", + "accountKey": { + "type": "SecureString", + "value": "[parameters('traininglake01noneKV_accountKey')]" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "Managed", + "typeProperties": { + "computeProperties": { + "location": "UK South", + "dataFlowProperties": { + "computeType": "General", + "coreCount": 8, + "timeToLive": 240 + } + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/PaulsLaptopIR')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "SelfHosted", + "typeProperties": { + "linkedInfo": { + "resourceId": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]", + "authorizationType": "Rbac" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "Managed", + "typeProperties": { + "computeProperties": { + "location": "UK South", + "dataFlowProperties": { + "computeType": "General", + "coreCount": 8, + "timeToLive": 0, + "cleanup": true + } + } + }, + "managedVirtualNetwork": { + "type": "ManagedVirtualNetworkReference", + "referenceName": "default" + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/MappingOrderAggregation')]", + "type": "Microsoft.DataFactory/factories/dataflows", + "apiVersion": "2018-06-01", + "properties": { + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "LakeFileOrderHeaderParquet", + "type": "DatasetReference" + }, + "name": "OrderHeader" + }, + { + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + }, + "name": "OrderLineDetails" + } + ], + "sinks": [ + { + "dataset": { + "referenceName": "TableOrderSummary", + "type": "DatasetReference" + }, + "name": "OrderSummary" + } + ], + "transformations": [ + { + "name": "JoinHeaderToLineDetails" + }, + { + "name": "OrderLineCount" + } + ], + "script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary" + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LakeFileOrderHeaderParquet')]", + "[concat(variables('factoryId'), '/datasets/LakeFileOrderDetailLinesParquet')]", + "[concat(variables('factoryId'), '/datasets/TableOrderSummary')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]", + "type": "Microsoft.DataFactory/factories/dataflows", + "apiVersion": "2018-06-01", + "properties": { + "type": "WranglingDataFlow", + "typeProperties": { + "sources": [ + { + "name": "LakeFileOrderDetailLinesParquet", + "script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet", + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + } + } + ], + "script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n" + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LakeFileOrderDetailLinesParquet')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingCredential01')]", + "type": "Microsoft.DataFactory/factories/credentials", + "apiVersion": "2018-06-01", + "properties": { + "type": "ManagedIdentity", + "typeProperties": { + "resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/default')]", + "type": "Microsoft.DataFactory/factories/managedVirtualNetworks", + "apiVersion": "2018-06-01", + "properties": {}, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", + "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", + "apiVersion": "2018-06-01", + "properties": { + "privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]", + "groupId": "[parameters('AzureKeyVault1_properties_groupId')]", + "fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" + ] + } + ] +} \ No newline at end of file diff --git a/TrainingFactoryDev/ARMTemplateParametersForFactory.json b/TrainingFactoryDev/ARMTemplateParametersForFactory.json new file mode 100644 index 0000000..0a533a8 --- /dev/null +++ b/TrainingFactoryDev/ARMTemplateParametersForFactory.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "value": "TrainingFactoryDev" + }, + "TrainingStore01_connectionString": { + "value": "" + }, + "traininglak01_accountKey": { + "value": "" + }, + "traininglake01noneKV_accountKey": { + "value": "" + }, + "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { + "value": "@linkedService().DBConnectionSecret" + }, + "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { + "value": "@linkedService().KeyVaultSecretName" + }, + "BatchForTraining01_properties_typeProperties_batchUri": { + "value": "https://batchfortraining01.uksouth.batch.azure.com" + }, + "BatchForTraining01_properties_typeProperties_poolName": { + "value": "09a2ec47-0cc8-4492-935c-04852e3ec654" + }, + "BatchForTraining01_properties_typeProperties_accountName": { + "value": "batchfortraining01" + }, + "GenericKeys_properties_typeProperties_baseUrl": { + "value": "@{linkedService().baseUrl}" + }, + "LaptopFiles_properties_typeProperties_host": { + "value": "C:\\ADFRoot\\" + }, + "LaptopFiles_properties_typeProperties_userId": { + "value": "ADFIRUser" + }, + "TrainingKeys01_properties_typeProperties_baseUrl": { + "value": "https://TrainingKeys01.vault.azure.net/" + }, + "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { + "value": "https://TrainingKeys01.vault.azure.net/" + }, + "trainingdb01_properties_typeProperties_connectionString_secretName": { + "value": "ConnectionString-trainingdb01" + }, + "traininglak01_properties_typeProperties_url": { + "value": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01_properties_typeProperties_url": { + "value": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01noneKV_properties_typeProperties_url": { + "value": "https://traininglake01.dfs.core.windows.net" + }, + "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { + "value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" + }, + "AzureKeyVault1_properties_privateLinkResourceId": { + "value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" + }, + "AzureKeyVault1_properties_groupId": { + "value": "vault" + }, + "AzureKeyVault1_properties_fqdns": { + "value": [ + "trainingkeys01.vaultcore.azure.net" + ] + } + } +} \ No newline at end of file diff --git a/TrainingFactoryDev/globalParameters/TrainingFactoryDev_GlobalParameters.json b/TrainingFactoryDev/globalParameters/TrainingFactoryDev_GlobalParameters.json new file mode 100644 index 0000000..2e6e5ac --- /dev/null +++ b/TrainingFactoryDev/globalParameters/TrainingFactoryDev_GlobalParameters.json @@ -0,0 +1,10 @@ +{ + "Environment": { + "type": "string", + "value": "Dev" + }, + "KeyVaultName": { + "type": "string", + "value": "trainingkeys01" + } +} \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json b/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json new file mode 100644 index 0000000..15cdd52 --- /dev/null +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "value": "TrainingFactoryDev" + }, + "TrainingStore01_connectionString": { + "value": "" + }, + "traininglak01_accountKey": { + "value": "" + }, + "traininglake01noneKV_accountKey": { + "value": "" + }, + "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { + "value": "@linkedService().DBConnectionSecret" + }, + "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { + "value": "@linkedService().KeyVaultSecretName" + }, + "BatchForTraining01_properties_typeProperties_batchUri": { + "value": "https://batchfortraining01.uksouth.batch.azure.com" + }, + "BatchForTraining01_properties_typeProperties_poolName": { + "value": "09a2ec47-0cc8-4492-935c-04852e3ec654" + }, + "BatchForTraining01_properties_typeProperties_accountName": { + "value": "batchfortraining01" + }, + "GenericKeys_properties_typeProperties_baseUrl": { + "value": "@{linkedService().baseUrl}" + }, + "LaptopFiles_properties_typeProperties_host": { + "value": "C:\\ADFRoot\\" + }, + "LaptopFiles_properties_typeProperties_userId": { + "value": "ADFIRUser" + }, + "TrainingKeys01_properties_typeProperties_baseUrl": { + "value": "https://TrainingKeys01.vault.azure.net/" + }, + "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { + "value": "https://TrainingKeys01.vault.azure.net/" + }, + "trainingdb01_properties_typeProperties_connectionString_secretName": { + "value": "ConnectionString-trainingdb01" + }, + "traininglak01_properties_typeProperties_url": { + "value": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01_properties_typeProperties_url": { + "value": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01noneKV_properties_typeProperties_url": { + "value": "https://traininglake01.dfs.core.windows.net" + }, + "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { + "value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" + }, + "AzureKeyVault1_properties_privateLinkResourceId": { + "value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" + }, + "AzureKeyVault1_properties_groupId": { + "value": "vault" + }, + "AzureKeyVault1_properties_fqdns": { + "value": [ + "trainingkeys01.vaultcore.azure.net" + ] + }, + "containerUri": { + "value": "" + }, + "containerSasToken": { + "value": "" + } + } +} \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json new file mode 100644 index 0000000..f76a045 --- /dev/null +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json @@ -0,0 +1,1013 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name", + "defaultValue": "TrainingFactoryDev" + }, + "TrainingStore01_connectionString": { + "type": "secureString", + "metadata": "Secure string for 'connectionString' of 'TrainingStore01'" + }, + "traininglak01_accountKey": { + "type": "secureString", + "metadata": "Secure string for 'accountKey' of 'traininglak01'" + }, + "traininglake01noneKV_accountKey": { + "type": "secureString", + "metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'" + }, + "GenericKeys_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "@{linkedService().baseUrl}" + }, + "TrainingKeys01_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "https://TrainingKeys01.vault.azure.net/" + }, + "traininglak01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01noneKV_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { + "type": "string", + "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" + }, + "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "@linkedService().DBConnectionSecret" + }, + "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "@linkedService().KeyVaultSecretName" + }, + "BatchForTraining01_properties_typeProperties_batchUri": { + "type": "string", + "defaultValue": "https://batchfortraining01.uksouth.batch.azure.com" + }, + "BatchForTraining01_properties_typeProperties_poolName": { + "type": "string", + "defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654" + }, + "BatchForTraining01_properties_typeProperties_accountName": { + "type": "string", + "defaultValue": "batchfortraining01" + }, + "LaptopFiles_properties_typeProperties_host": { + "type": "string", + "defaultValue": "C:\\ADFRoot\\" + }, + "LaptopFiles_properties_typeProperties_userId": { + "type": "string", + "defaultValue": "ADFIRUser" + }, + "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "https://TrainingKeys01.vault.azure.net/" + }, + "trainingdb01_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "traininglake01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/1950 Activities')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "50", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Wait 1 Minute 1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 2", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 3", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 4", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 5", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 6", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 7", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 8", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 9", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 10", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 11", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 12", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 13", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 14", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 15", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 16", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 17", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 18", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 19", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 20", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 21", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 22", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 23", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 24", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 25", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 26", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 27", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 28", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 29", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 30", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 31", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 32", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 33", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 34", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 35", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 36", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 37", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 38", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 39", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [], + "lastPublishTime": "2022-02-03T14:39:03Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 10')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Wait 10", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 10 + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/GenericKeys')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "description": "https://TrainingKeys01.vault.azure.net", + "parameters": { + "baseUrl": { + "type": "String" + } + }, + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingKeys01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingStore01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobStorage", + "typeProperties": { + "connectionString": "[parameters('TrainingStore01_connectionString')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/traininglak01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglak01_properties_typeProperties_url')]", + "accountKey": { + "type": "SecureString", + "value": "[parameters('traininglak01_accountKey')]" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/traininglake01noneKV')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglake01noneKV_properties_typeProperties_url')]", + "accountKey": { + "type": "SecureString", + "value": "[parameters('traininglake01noneKV_accountKey')]" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "Managed", + "typeProperties": { + "computeProperties": { + "location": "UK South", + "dataFlowProperties": { + "computeType": "General", + "coreCount": 8, + "timeToLive": 240 + } + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/PaulsLaptopIR')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "SelfHosted", + "typeProperties": { + "linkedInfo": { + "resourceId": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]", + "authorizationType": "Rbac" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingCredential01')]", + "type": "Microsoft.DataFactory/factories/credentials", + "apiVersion": "2018-06-01", + "properties": { + "type": "ManagedIdentity", + "typeProperties": { + "resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/default')]", + "type": "Microsoft.DataFactory/factories/managedVirtualNetworks", + "apiVersion": "2018-06-01", + "properties": {}, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01noneKV", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "OrderDetailLines.parquet", + "folderPath": "Raw", + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [ + { + "name": "SalesOrderID", + "type": "INT32" + }, + { + "name": "SalesOrderDetailID", + "type": "INT32" + }, + { + "name": "OrderQty", + "type": "INT32" + }, + { + "name": "ProductID", + "type": "INT32" + }, + { + "name": "UnitPrice", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "UnitPriceDiscount", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "LineTotal", + "type": "DECIMAL", + "precision": 38, + "scale": 6 + }, + { + "name": "rowguid", + "type": "UTF8" + }, + { + "name": "ModifiedDate", + "type": "INT96" + } + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileOrderHeaderParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01noneKV", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "OrderHeader.parquet", + "folderPath": "Raw", + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [ + { + "name": "SalesOrderID", + "type": "INT32" + }, + { + "name": "RevisionNumber", + "type": "INT32" + }, + { + "name": "OrderDate", + "type": "INT96" + }, + { + "name": "DueDate", + "type": "INT96" + }, + { + "name": "ShipDate", + "type": "INT96" + }, + { + "name": "Status", + "type": "INT32" + }, + { + "name": "OnlineOrderFlag", + "type": "BOOLEAN" + }, + { + "name": "SalesOrderNumber", + "type": "UTF8" + }, + { + "name": "PurchaseOrderNumber", + "type": "UTF8" + }, + { + "name": "AccountNumber", + "type": "UTF8" + }, + { + "name": "CustomerID", + "type": "INT32" + }, + { + "name": "ShipToAddressID", + "type": "INT32" + }, + { + "name": "BillToAddressID", + "type": "INT32" + }, + { + "name": "ShipMethod", + "type": "UTF8" + }, + { + "name": "CreditCardApprovalCode", + "type": "UTF8" + }, + { + "name": "SubTotal", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "TaxAmt", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "Freight", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "TotalDue", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "Comment", + "type": "UTF8" + }, + { + "name": "rowguid", + "type": "UTF8" + }, + { + "name": "ModifiedDate", + "type": "INT96" + } + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "parameters": { + "DBConnectionSecret": { + "type": "string" + } + }, + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": { + "value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]", + "type": "Expression" + } + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "parameters": { + "KeyVaultURL": { + "type": "string" + }, + "KeyVaultSecretName": { + "type": "string" + } + }, + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "GenericKeys", + "type": "LinkedServiceReference", + "parameters": { + "baseUrl": { + "value": "@linkedService().KeyVaultURL", + "type": "Expression" + } + } + }, + "secretName": { + "value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]", + "type": "Expression" + } + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/GenericKeys')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/BatchForTraining01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBatch", + "typeProperties": { + "batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]", + "poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]", + "accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]", + "linkedServiceName": { + "referenceName": "TrainingStore01", + "type": "LinkedServiceReference" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFiles')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "FileServer", + "typeProperties": { + "host": "[parameters('LaptopFiles_properties_typeProperties_host')]", + "userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]", + "password": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "ADFIRUser" + } + }, + "connectVia": { + "referenceName": "PaulsLaptopIR", + "type": "IntegrationRuntimeReference" + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]", + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]", + "credential": { + "referenceName": "TrainingCredential01", + "type": "CredentialReference" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/credentials/TrainingCredential01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/trainingdb01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/traininglake01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglake01_properties_typeProperties_url')]", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "traininglake01" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" + ] + } + ] +} \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json new file mode 100644 index 0000000..aac9873 --- /dev/null +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json @@ -0,0 +1,954 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name", + "defaultValue": "TrainingFactoryDev" + }, + "AzureKeyVault1_properties_privateLinkResourceId": { + "type": "string", + "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" + }, + "AzureKeyVault1_properties_groupId": { + "type": "string", + "defaultValue": "vault" + }, + "AzureKeyVault1_properties_fqdns": { + "type": "array", + "defaultValue": [ + "trainingkeys01.vaultcore.azure.net" + ] + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "Managed", + "typeProperties": { + "computeProperties": { + "location": "UK South", + "dataFlowProperties": { + "computeType": "General", + "coreCount": 8, + "timeToLive": 0, + "cleanup": true + } + } + }, + "managedVirtualNetwork": { + "type": "ManagedVirtualNetworkReference", + "referenceName": "default" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", + "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", + "apiVersion": "2018-06-01", + "properties": { + "privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]", + "groupId": "[parameters('AzureKeyVault1_properties_groupId')]", + "fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Custom Activity')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "HelloWorld", + "type": "Custom", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "command": "CustomActivity.exe", + "resourceLinkedService": { + "referenceName": "TrainingStore01", + "type": "LinkedServiceReference" + }, + "folderPath": "customactivities", + "referenceObjects": { + "linkedServices": [], + "datasets": [] + } + }, + "linkedServiceName": { + "referenceName": "BatchForTraining01", + "type": "LinkedServiceReference" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [], + "lastPublishTime": "2022-02-03T14:39:04Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnection", + "type": "LinkedServiceReference", + "parameters": { + "DBConnectionSecret": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnectionFromAnyKeyVault", + "type": "LinkedServiceReference", + "parameters": { + "KeyVaultURL": { + "value": "@dataset().KeyVaultURL", + "type": "Expression" + }, + "KeyVaultSecretName": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + }, + "KeyVaultURL": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": {} + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "Person.csv", + "folderPath": "Landing", + "fileSystem": "datawarehouse" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [ + { + "type": "String" + }, + { + "type": "String" + }, + { + "type": "String" + } + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFiles')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": "Person.csv", + "folderPath": "ForUpload/People" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [ + { + "type": "String" + }, + { + "type": "String" + }, + { + "type": "String" + } + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFolders')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopsFiles')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/TableOrderSummary')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [ + { + "name": "SalesOrderNumber", + "type": "varchar" + }, + { + "name": "RecordCount", + "type": "int", + "precision": 10 + } + ], + "typeProperties": { + "schema": "dbo", + "table": "OrderSummary" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]", + "type": "Microsoft.DataFactory/factories/dataflows", + "apiVersion": "2018-06-01", + "properties": { + "type": "WranglingDataFlow", + "typeProperties": { + "sources": [ + { + "name": "LakeFileOrderDetailLinesParquet", + "script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet", + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + } + } + ], + "script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Upload Person", + "description": "hfdhgfdhfd", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopFilePersonCSV", + "type": "DatasetReference", + "parameters": {} + } + ], + "outputs": [ + { + "referenceName": "LakeFilePersonCSV", + "type": "DatasetReference", + "parameters": {} + } + ] + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-29T13:50:28Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]", + "[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Upload Any File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": "Person.csv" + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": "Person.csv" + } + } + ] + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:46Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", + "[concat(variables('factoryId'), '/datasets/LakeFiles')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "GetMetadata", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "LaptopFolders", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People" + } + }, + "fieldList": [ + "childItems" + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + } + }, + { + "name": "Upload Files", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.childItems", + "type": "Expression" + }, + "activities": [ + { + "name": "Upload File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:45Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/LaptopFolders')]", + "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", + "[concat(variables('factoryId'), '/datasets/LakeFiles')]" + ] + } + ] +} \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json new file mode 100644 index 0000000..7ca9a87 --- /dev/null +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json @@ -0,0 +1,1270 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name", + "defaultValue": "TrainingFactoryDev" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "UploadFiles", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Log Upload Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy By File Type", + "type": "Switch", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@substring(item().FileName,add(indexof(item().FileName,'.'),1),sub(length(item().FileName),add(indexof(item().FileName,'.'),1)))", + "type": "Expression" + }, + "cases": [ + { + "value": "csv", + "activities": [ + { + "name": "CSV Copy", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "additionalColumns": [ + { + "name": "FilePath", + "value": "$$FILEPATH" + }, + { + "name": "Environment", + "value": { + "value": "@pipeline().globalParameters.Environment", + "type": "Expression" + } + }, + { + "name": "PipelineName", + "value": { + "value": "@pipeline().Pipeline", + "type": "Expression" + } + }, + { + "name": "RunId", + "value": { + "value": "@pipeline().RunId", + "type": "Expression" + } + }, + { + "name": "RunDate", + "value": { + "value": "@utcnow()", + "type": "Expression" + } + } + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": false, + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + }, + { + "value": "parquet", + "activities": [ + { + "name": "Parquet Copy", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "ParquetSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": false, + "enablePartitionDiscovery": false + } + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "ParquetWriteSettings" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopFilesParquet", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFileParquet", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + } + ], + "defaultActivities": [ + { + "name": "Binary Copy", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": false + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "LaptopFilesBinary", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFilesBinary", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ] + } + }, + { + "name": "Log Upload Start", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[LogUploadStart]", + "storedProcedureParameters": { + "FileCount": { + "type": "Int32", + "value": { + "value": "@activity('Get File List').output.count", + "type": "Expression" + } + }, + "TriggerId": { + "type": "String", + "value": { + "value": "@pipeline().TriggerId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Log Upload Finish", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "UploadFiles", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[LogUploadFinish]", + "storedProcedureParameters": { + "LogId": { + "value": { + "value": "@activity('Log Upload Start').output.firstRow.LogId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "variables": { + "DemoVariable": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:14Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/05 - Get File List Utility')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.GetFileQuery", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "GetFileQuery": { + "type": "string" + } + }, + "variables": { + "DemoVariable": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:14Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Secret", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Set Key URL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": { + "value": "@variables('CompleteSecretURL')", + "type": "Expression" + }, + "method": "GET", + "headers": {}, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set Key URL", + "type": "SetVariable", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "variableName": "CompleteSecretURL", + "value": { + "value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')", + "type": "Expression" + } + } + }, + { + "name": "Nested Key Vault URL", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get Secret", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTableAnyKeyVault", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": "ConnectionString-trainingdb01", + "SchemaName": "sys", + "TableName": "objects", + "KeyVaultURL": { + "value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')", + "type": "Expression" + } + } + } + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "SecretName": { + "type": "string", + "defaultValue": "DemoKeyGetWithWebActivity" + } + }, + "variables": { + "CompleteSecretURL": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Lazy Replication')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Table List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TableLookupQuery", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": "sys", + "TableName": "objects" + } + }, + "firstRowOnly": false + } + }, + { + "name": "Copy Tables", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Table List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Table List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy Table", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.TargetConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "SourceConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "TargetConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb02" + }, + "TableLookupQuery": { + "type": "string", + "defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'" + } + }, + "folder": { + "name": "Demo Pipelines/Dynamic Linked Services" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:43:42Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Bucket Contents", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]", + "storedProcedureParameters": { + "BucketId": { + "type": "Int32", + "value": { + "value": "@pipeline().parameters.BucketId", + "type": "Expression" + } + } + }, + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Execute Processes", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Contents", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Contents').output.value", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Run Process", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": { + "value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "BucketId": { + "type": "int" + } + }, + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/MappingOrderAggregation')]", + "type": "Microsoft.DataFactory/factories/dataflows", + "apiVersion": "2018-06-01", + "properties": { + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "LakeFileOrderHeaderParquet", + "type": "DatasetReference" + }, + "name": "OrderHeader" + }, + { + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + }, + "name": "OrderLineDetails" + } + ], + "sinks": [ + { + "dataset": { + "referenceName": "TableOrderSummary", + "type": "DatasetReference" + }, + "name": "OrderSummary" + } + ], + "transformations": [ + { + "name": "JoinHeaderToLineDetails" + }, + { + "name": "OrderLineCount" + } + ], + "script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/06 - Example Global Param')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "06 - Get Any File List", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "05 - Get File List Utility", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "GetFileQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;" + } + } + }, + { + "name": "Set Pipe Output", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "06 - Get Any File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "WorkerOutput", + "value": { + "value": "@activity('06 - Get Any File List').output.pipelineRunId", + "type": "Expression" + } + } + }, + { + "name": "Set Global Param", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "06 - Get Any File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "GlobalParam", + "value": { + "value": "@concat(pipeline().globalParameters.Environment,'Test')", + "type": "Expression" + } + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "variables": { + "WorkerOutput": { + "type": "String" + }, + "GlobalParam": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/05 - Get File List Utility')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Order Summary with Mapping')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Mapping Order Aggregation", + "type": "ExecuteDataFlow", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataflow": { + "referenceName": "MappingOrderAggregation", + "type": "DataFlowReference", + "parameters": {}, + "datasetParameters": { + "OrderHeader": {}, + "OrderLineDetails": {}, + "OrderSummary": {} + } + }, + "staging": {}, + "compute": { + "coreCount": 8, + "computeType": "General" + }, + "traceLevel": "Fine" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Flows" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:15Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/dataflows/MappingOrderAggregation')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Scale Out Level 1')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Bucket Ids", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Reset And Log Start", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Reset Bucket Process Map", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[GetBuckets]", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Call Buckets", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Ids", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Ids').output.value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 50, + "activities": [ + { + "name": "Call Level 2", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "Scale Out Level 2", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "BucketId": { + "value": "@{item().BucketId}", + "type": "Expression" + } + } + } + } + ] + } + }, + { + "name": "Log End", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Call Buckets", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Reset And Log Start", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[SetBucketLogEntry]", + "storedProcedureParameters": { + "LogId": { + "value": { + "value": "@activity('Reset And Log Start').output.firstRow.LogId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + }, + { + "name": "Reset And Log Start", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[SetBucketLogEntry]", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Reset Bucket Process Map", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[SetBucketProcesses]" + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/Scale Out Level 2')]" + ] + } + ] +} \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json new file mode 100644 index 0000000..d389a93 --- /dev/null +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json @@ -0,0 +1,231 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name", + "defaultValue": "TrainingFactoryDev" + }, + "TrainingStore01_connectionString": { + "type": "secureString", + "metadata": "Secure string for 'connectionString' of 'TrainingStore01'" + }, + "traininglak01_accountKey": { + "type": "secureString", + "metadata": "Secure string for 'accountKey' of 'traininglak01'" + }, + "traininglake01noneKV_accountKey": { + "type": "secureString", + "metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'" + }, + "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "@linkedService().DBConnectionSecret" + }, + "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "@linkedService().KeyVaultSecretName" + }, + "BatchForTraining01_properties_typeProperties_batchUri": { + "type": "string", + "defaultValue": "https://batchfortraining01.uksouth.batch.azure.com" + }, + "BatchForTraining01_properties_typeProperties_poolName": { + "type": "string", + "defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654" + }, + "BatchForTraining01_properties_typeProperties_accountName": { + "type": "string", + "defaultValue": "batchfortraining01" + }, + "GenericKeys_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "@{linkedService().baseUrl}" + }, + "LaptopFiles_properties_typeProperties_host": { + "type": "string", + "defaultValue": "C:\\ADFRoot\\" + }, + "LaptopFiles_properties_typeProperties_userId": { + "type": "string", + "defaultValue": "ADFIRUser" + }, + "TrainingKeys01_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "https://TrainingKeys01.vault.azure.net/" + }, + "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "https://TrainingKeys01.vault.azure.net/" + }, + "trainingdb01_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "traininglak01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "traininglake01noneKV_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, + "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { + "type": "string", + "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" + }, + "AzureKeyVault1_properties_privateLinkResourceId": { + "type": "string", + "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" + }, + "AzureKeyVault1_properties_groupId": { + "type": "string", + "defaultValue": "vault" + }, + "AzureKeyVault1_properties_fqdns": { + "type": "array", + "defaultValue": [ + "trainingkeys01.vaultcore.azure.net" + ] + }, + "containerUri": { + "type": "string", + "metadata": "URI to storage account containing linked ARM templates" + }, + "containerSasToken": { + "type": "string", + "metadata": "Sas token to storage account containing linked ARM templates" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "TrainingFactoryDev_ArmTemplate_0", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-02-01", + "properties": { + "mode": "incremental", + "templateLink": { + "uri": "[concat(parameters('containerUri'), '/ArmTemplate_0.json', parameters('containerSasToken'))]", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "factoryName": { + "value": "[parameters('factoryName')]" + }, + "TrainingStore01_connectionString": { + "value": "[parameters('TrainingStore01_connectionString')]" + }, + "traininglak01_accountKey": { + "value": "[parameters('traininglak01_accountKey')]" + }, + "traininglake01noneKV_accountKey": { + "value": "[parameters('traininglake01noneKV_accountKey')]" + }, + "GenericKeys_properties_typeProperties_baseUrl": { + "value": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]" + }, + "TrainingKeys01_properties_typeProperties_baseUrl": { + "value": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]" + }, + "traininglak01_properties_typeProperties_url": { + "value": "[parameters('traininglak01_properties_typeProperties_url')]" + }, + "traininglake01noneKV_properties_typeProperties_url": { + "value": "[parameters('traininglake01noneKV_properties_typeProperties_url')]" + }, + "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { + "value": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]" + }, + "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { + "value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]" + }, + "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { + "value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]" + }, + "BatchForTraining01_properties_typeProperties_batchUri": { + "value": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]" + }, + "BatchForTraining01_properties_typeProperties_poolName": { + "value": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]" + }, + "BatchForTraining01_properties_typeProperties_accountName": { + "value": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]" + }, + "LaptopFiles_properties_typeProperties_host": { + "value": "[parameters('LaptopFiles_properties_typeProperties_host')]" + }, + "LaptopFiles_properties_typeProperties_userId": { + "value": "[parameters('LaptopFiles_properties_typeProperties_userId')]" + }, + "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { + "value": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]" + }, + "trainingdb01_properties_typeProperties_connectionString_secretName": { + "value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" + }, + "traininglake01_properties_typeProperties_url": { + "value": "[parameters('traininglake01_properties_typeProperties_url')]" + } + } + }, + "dependsOn": [] + }, + { + "name": "TrainingFactoryDev_ArmTemplate_1", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-02-01", + "properties": { + "mode": "incremental", + "templateLink": { + "uri": "[concat(parameters('containerUri'), '/ArmTemplate_1.json', parameters('containerSasToken'))]", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "factoryName": { + "value": "[parameters('factoryName')]" + }, + "AzureKeyVault1_properties_privateLinkResourceId": { + "value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]" + }, + "AzureKeyVault1_properties_groupId": { + "value": "[parameters('AzureKeyVault1_properties_groupId')]" + }, + "AzureKeyVault1_properties_fqdns": { + "value": "[parameters('AzureKeyVault1_properties_fqdns')]" + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'TrainingFactoryDev_ArmTemplate_0')]" + ] + }, + { + "name": "TrainingFactoryDev_ArmTemplate_2", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-02-01", + "properties": { + "mode": "incremental", + "templateLink": { + "uri": "[concat(parameters('containerUri'), '/ArmTemplate_2.json', parameters('containerSasToken'))]", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "factoryName": { + "value": "[parameters('factoryName')]" + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'TrainingFactoryDev_ArmTemplate_1')]" + ] + } + ] +} \ No newline at end of file