From 2a23b67be208f65074a93e4c871b5a2bc71e0367 Mon Sep 17 00:00:00 2001 From: Paul Andrew <60190428+mrpaulandrew@users.noreply.github.com> Date: Wed, 9 Mar 2022 15:59:39 +0000 Subject: [PATCH] ARM template and parameters deployed on 3-9-2022 15:59:37, based on the collaboration branch's commit ID: 09c848bc7d65f8fe7a0f7efad419136c1941ef5d --- TrainingFactoryDev/ARMTemplateForFactory.json | 24 +++ .../linkedTemplates/ArmTemplate_0.json | 51 +++-- .../linkedTemplates/ArmTemplate_1.json | 188 ++++-------------- .../linkedTemplates/ArmTemplate_2.json | 143 +++++++++++++ .../linkedTemplates/ArmTemplate_master.json | 6 +- 5 files changed, 231 insertions(+), 181 deletions(-) diff --git a/TrainingFactoryDev/ARMTemplateForFactory.json b/TrainingFactoryDev/ARMTemplateForFactory.json index 89a7f92..f7036b9 100644 --- a/TrainingFactoryDev/ARMTemplateForFactory.json +++ b/TrainingFactoryDev/ARMTemplateForFactory.json @@ -3259,6 +3259,30 @@ "dependsOn": [ "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" ] + }, + { + "name": "[concat(parameters('factoryName'), '/pipeline1')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Wait1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 1 + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "annotations": [] + }, + "dependsOn": [] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json index c461722..0646d43 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json @@ -74,10 +74,6 @@ "trainingdb01_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "ConnectionString-trainingdb01" - }, - "traininglake01_properties_typeProperties_url": { - "type": "string", - "defaultValue": "https://traininglake01.dfs.core.windows.net" } }, "variables": { @@ -628,6 +624,30 @@ "properties": {}, "dependsOn": [] }, + { + "name": "[concat(parameters('factoryName'), '/pipeline1')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Wait1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 1 + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "annotations": [] + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", "type": "Microsoft.DataFactory/factories/datasets", @@ -985,29 +1005,6 @@ "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" ] - }, - { - "name": "[concat(parameters('factoryName'), '/traininglake01')]", - "type": "Microsoft.DataFactory/factories/linkedServices", - "apiVersion": "2018-06-01", - "properties": { - "annotations": [], - "type": "AzureBlobFS", - "typeProperties": { - "url": "[parameters('traininglake01_properties_typeProperties_url')]", - "accountKey": { - "type": "AzureKeyVaultSecret", - "store": { - "referenceName": "TrainingKeys01", - "type": "LinkedServiceReference" - }, - "secretName": "traininglake01" - } - } - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" - ] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json index 93be0d5..b9fb980 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json @@ -7,6 +7,10 @@ "metadata": "Data Factory name", "defaultValue": "TrainingFactoryDev" }, + "traininglake01_properties_typeProperties_url": { + "type": "string", + "defaultValue": "https://traininglake01.dfs.core.windows.net" + }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" @@ -26,6 +30,27 @@ "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ + { + "name": "[concat(parameters('factoryName'), '/traininglake01')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "[parameters('traininglake01_properties_typeProperties_url')]", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "traininglake01" + } + } + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", @@ -264,7 +289,9 @@ }, "schema": [] }, - "dependsOn": [] + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] }, { "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", @@ -303,7 +330,9 @@ } ] }, - "dependsOn": [] + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] }, { "name": "[concat(parameters('factoryName'), '/LakeFiles')]", @@ -346,7 +375,9 @@ }, "schema": [] }, - "dependsOn": [] + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] }, { "name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]", @@ -385,7 +416,9 @@ } } }, - "dependsOn": [] + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" + ] }, { "name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]", @@ -802,153 +835,6 @@ "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", "[concat(variables('factoryId'), '/datasets/LakeFiles')]" ] - }, - { - "name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "Get File List", - "type": "GetMetadata", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "dataset": { - "referenceName": "LaptopFolders", - "type": "DatasetReference", - "parameters": { - "Directory": "ForUpload\\People" - } - }, - "fieldList": [ - "childItems" - ], - "storeSettings": { - "type": "FileServerReadSettings", - "recursive": true - }, - "formatSettings": { - "type": "DelimitedTextReadSettings" - } - } - }, - { - "name": "Upload Files", - "type": "ForEach", - "dependsOn": [ - { - "activity": "Get File List", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "items": { - "value": "@activity('Get File List').output.childItems", - "type": "Expression" - }, - "activities": [ - { - "name": "Upload File", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "DelimitedTextSource", - "storeSettings": { - "type": "FileServerReadSettings", - "recursive": true - }, - "formatSettings": { - "type": "DelimitedTextReadSettings" - } - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "LaptopsFiles", - "type": "DatasetReference", - "parameters": { - "Directory": "ForUpload\\People", - "File": { - "value": "@item().name", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "LakeFiles", - "type": "DatasetReference", - "parameters": { - "Directory": "Landing", - "File": { - "value": "@item().name", - "type": "Expression" - } - } - } - ] - } - ] - } - } - ], - "policy": { - "elapsedTimeMetric": {}, - "cancelAfter": {} - }, - "folder": { - "name": "Demo Pipelines/Data Uploads" - }, - "annotations": [], - "lastPublishTime": "2020-08-27T09:06:45Z" - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/datasets/LaptopFolders')]", - "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", - "[concat(variables('factoryId'), '/datasets/LakeFiles')]" - ] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json index 7ca9a87..85db39e 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json @@ -12,6 +12,149 @@ "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ + { + "name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "GetMetadata", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "LaptopFolders", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People" + } + }, + "fieldList": [ + "childItems" + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + } + }, + { + "name": "Upload Files", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.childItems", + "type": "Expression" + }, + "activities": [ + { + "name": "Upload File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:45Z" + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]", "type": "Microsoft.DataFactory/factories/pipelines", diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json index d389a93..bd5423d 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json @@ -170,9 +170,6 @@ }, "trainingdb01_properties_typeProperties_connectionString_secretName": { "value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" - }, - "traininglake01_properties_typeProperties_url": { - "value": "[parameters('traininglake01_properties_typeProperties_url')]" } } }, @@ -192,6 +189,9 @@ "factoryName": { "value": "[parameters('factoryName')]" }, + "traininglake01_properties_typeProperties_url": { + "value": "[parameters('traininglake01_properties_typeProperties_url')]" + }, "AzureKeyVault1_properties_privateLinkResourceId": { "value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]" },