diff --git a/TrainingFactoryDev/ARMTemplateForFactory.json b/TrainingFactoryDev/ARMTemplateForFactory.json index ef3d4c9..f9fe40a 100644 --- a/TrainingFactoryDev/ARMTemplateForFactory.json +++ b/TrainingFactoryDev/ARMTemplateForFactory.json @@ -107,16 +107,13 @@ "type": "string", "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": { + "AzureFunctionTest_properties_privateLinkResourceId": { "type": "string", - "defaultValue": "adayfullofadf-sql.database.windows.net" + "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": { + "AzureFunctionTest_properties_groupId": { "type": "string", - "defaultValue": "sql-admin" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": { - "type": "secureString" + "defaultValue": "sites" }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", @@ -1480,7 +1477,7 @@ "cancelAfter": {} }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, @@ -1544,7 +1541,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, @@ -1625,7 +1622,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, @@ -1692,7 +1689,7 @@ "cancelAfter": {} }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, @@ -1701,6 +1698,189 @@ "[concat(variables('factoryId'), '/datasets/ADLS_BIN_AWProduct')]" ] }, + { + "name": "[concat(parameters('factoryName'), '/CopyAWUsingMetadata')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get catalog", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json", + "method": "GET", + "headers": {} + } + }, + { + "name": "Cache catalog as JSON array", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get catalog", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "Catalog", + "value": { + "value": "@json(activity('Get catalog').output.Response)", + "type": "Expression" + } + } + }, + { + "name": "ForEach AW entity", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Cache catalog as JSON array", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@variables('Catalog')", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy AW entity", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "ParquetWriteSettings" + } + }, + "enableStaging": false, + "translator": { + "value": "@item().translator", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "HTTP_TSV_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "EntityName": { + "value": "@item().EntityName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLS_PQT_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "EntityName": { + "value": "@item().EntityName", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Notify error", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Copy AW entity", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "SendEmail", + "method": "POST", + "headers": {}, + "body": { + "value": "{\n \"emailRecipients\": \"paul@mrpaulandrew.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "EmailSenderFunction", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "variables": { + "Catalog": { + "type": "Array", + "defaultValue": [] + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/HTTP_TSV_AdventureWorks')]", + "[concat(variables('factoryId'), '/datasets/ADLS_PQT_AdventureWorks')]", + "[concat(variables('factoryId'), '/linkedServices/EmailSenderFunction')]" + ] + }, { "name": "[concat(parameters('factoryName'), '/Custom Activity')]", "type": "Microsoft.DataFactory/factories/pipelines", @@ -2549,7 +2729,7 @@ "type": "LinkedServiceReference" }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -2581,7 +2761,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -2601,6 +2781,43 @@ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, + { + "name": "[concat(parameters('factoryName'), '/ADLS_PQT_AdventureWorks')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "parameters": { + "EntityName": { + "type": "string" + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@{dataset().EntityName}.parquet", + "type": "Expression" + }, + "folderPath": "Raw", + "fileSystem": "lakeroot" + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" + ] + }, { "name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", @@ -2616,7 +2833,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "DelimitedText", @@ -2772,7 +2989,7 @@ "type": "LinkedServiceReference" }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -2801,7 +3018,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -2819,6 +3036,43 @@ "[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]" ] }, + { + "name": "[concat(parameters('factoryName'), '/HTTP_TSV_AdventureWorks')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWGitHub", + "type": "LinkedServiceReference" + }, + "parameters": { + "EntityName": { + "type": "string" + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "HttpServerLocation", + "relativeUrl": { + "value": "@{dataset().EntityName}.csv", + "type": "Expression" + } + }, + "columnDelimiter": "\t", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]" + ] + }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", "type": "Microsoft.DataFactory/factories/datasets", @@ -3650,12 +3904,12 @@ } }, "connectVia": { - "referenceName": "PaulsLaptopIR", + "referenceName": "Laptop", "type": "IntegrationRuntimeReference" } }, "dependsOn": [ - "[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]", + "[concat(variables('factoryId'), '/integrationRuntimes/Laptop')]", "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" ] }, @@ -3783,6 +4037,26 @@ }, "dependsOn": [] }, + { + "name": "[concat(parameters('factoryName'), '/trigger1')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 15, + "startTime": "2023-02-06T14:56:00Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", @@ -3817,36 +4091,6 @@ }, "dependsOn": [] }, - { - "name": "[concat(parameters('factoryName'), '/SSISIntegrationRuntime')]", - "type": "Microsoft.DataFactory/factories/integrationRuntimes", - "apiVersion": "2018-06-01", - "properties": { - "type": "Managed", - "typeProperties": { - "computeProperties": { - "location": "UK South", - "nodeSize": "Standard_D2_v3", - "numberOfNodes": 1, - "maxParallelExecutionsPerNode": 2 - }, - "ssisProperties": { - "catalogInfo": { - "catalogServerEndpoint": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]", - "catalogAdminUserName": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]", - "catalogAdminPassword": { - "type": "SecureString", - "value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]" - }, - "catalogPricingTier": "S1" - }, - "edition": "Standard", - "licenseType": "LicenseIncluded" - } - } - }, - "dependsOn": [] - }, { "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", @@ -4363,6 +4607,18 @@ "properties": {}, "dependsOn": [] }, + { + "name": "[concat(parameters('factoryName'), '/default/AzureFunctionTest')]", + "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", + "apiVersion": "2018-06-01", + "properties": { + "privateLinkResourceId": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]", + "groupId": "[parameters('AzureFunctionTest_properties_groupId')]" + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" + ] + }, { "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", @@ -4375,6 +4631,16 @@ "dependsOn": [ "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" ] + }, + { + "name": "[concat(parameters('factoryName'), '/Laptop')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "SelfHosted", + "typeProperties": {} + }, + "dependsOn": [] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/ARMTemplateParametersForFactory.json b/TrainingFactoryDev/ARMTemplateParametersForFactory.json index 51a917e..c39b8f4 100644 --- a/TrainingFactoryDev/ARMTemplateParametersForFactory.json +++ b/TrainingFactoryDev/ARMTemplateParametersForFactory.json @@ -80,14 +80,11 @@ "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { "value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": { - "value": "adayfullofadf-sql.database.windows.net" + "AzureFunctionTest_properties_privateLinkResourceId": { + "value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": { - "value": "sql-admin" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": { - "value": "" + "AzureFunctionTest_properties_groupId": { + "value": "sites" }, "AzureKeyVault1_properties_privateLinkResourceId": { "value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json b/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json index 9f14fb1..14df814 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplateParameters_master.json @@ -80,14 +80,11 @@ "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { "value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": { - "value": "adayfullofadf-sql.database.windows.net" + "AzureFunctionTest_properties_privateLinkResourceId": { + "value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": { - "value": "sql-admin" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": { - "value": "" + "AzureFunctionTest_properties_groupId": { + "value": "sites" }, "AzureKeyVault1_properties_privateLinkResourceId": { "value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json index ced6245..508aafb 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json @@ -62,17 +62,6 @@ "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { "type": "string", "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": { - "type": "string", - "defaultValue": "adayfullofadf-sql.database.windows.net" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": { - "type": "string", - "defaultValue": "sql-admin" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": { - "type": "secureString" } }, "variables": { @@ -644,6 +633,26 @@ }, "dependsOn": [] }, + { + "name": "[concat(parameters('factoryName'), '/trigger1')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 15, + "startTime": "2023-02-06T14:56:00Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", @@ -678,36 +687,6 @@ }, "dependsOn": [] }, - { - "name": "[concat(parameters('factoryName'), '/SSISIntegrationRuntime')]", - "type": "Microsoft.DataFactory/factories/integrationRuntimes", - "apiVersion": "2018-06-01", - "properties": { - "type": "Managed", - "typeProperties": { - "computeProperties": { - "location": "UK South", - "nodeSize": "Standard_D2_v3", - "numberOfNodes": 1, - "maxParallelExecutionsPerNode": 2 - }, - "ssisProperties": { - "catalogInfo": { - "catalogServerEndpoint": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]", - "catalogAdminUserName": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]", - "catalogAdminPassword": { - "type": "SecureString", - "value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]" - }, - "catalogPricingTier": "S1" - }, - "edition": "Standard", - "licenseType": "LicenseIncluded" - } - } - }, - "dependsOn": [] - }, { "name": "[concat(parameters('factoryName'), '/TrainingCredential01')]", "type": "Microsoft.DataFactory/factories/credentials", @@ -727,6 +706,16 @@ "properties": {}, "dependsOn": [] }, + { + "name": "[concat(parameters('factoryName'), '/Laptop')]", + "type": "Microsoft.DataFactory/factories/integrationRuntimes", + "apiVersion": "2018-06-01", + "properties": { + "type": "SelfHosted", + "typeProperties": {} + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/EmailSender')]", "type": "Microsoft.DataFactory/factories/pipelines", @@ -798,7 +787,7 @@ "type": "LinkedServiceReference" }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -811,39 +800,6 @@ "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/HTTP_AWProduct')]" ] - }, - { - "name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]", - "type": "Microsoft.DataFactory/factories/datasets", - "apiVersion": "2018-06-01", - "properties": { - "linkedServiceName": { - "referenceName": "HTTP_AWGitHub", - "type": "LinkedServiceReference" - }, - "parameters": { - "Entity": { - "type": "string" - } - }, - "folder": { - "name": "Labs" - }, - "annotations": [], - "type": "Binary", - "typeProperties": { - "location": { - "type": "HttpServerLocation", - "relativeUrl": { - "value": "@{dataset().Entity}.csv", - "type": "Expression" - } - } - } - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]" - ] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json index f2b35b9..fd1c7f6 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json @@ -51,6 +51,14 @@ "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, + "AzureFunctionTest_properties_privateLinkResourceId": { + "type": "string", + "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions" + }, + "AzureFunctionTest_properties_groupId": { + "type": "string", + "defaultValue": "sites" + }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" @@ -70,6 +78,72 @@ "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ + { + "name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWGitHub", + "type": "LinkedServiceReference" + }, + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Labs1" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "HttpServerLocation", + "relativeUrl": { + "value": "@{dataset().Entity}.csv", + "type": "Expression" + } + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/HTTP_TSV_AdventureWorks')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWGitHub", + "type": "LinkedServiceReference" + }, + "parameters": { + "EntityName": { + "type": "string" + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "HttpServerLocation", + "relativeUrl": { + "value": "@{dataset().EntityName}.csv", + "type": "Expression" + } + }, + "columnDelimiter": "\t", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", "type": "Microsoft.DataFactory/factories/datasets", @@ -389,7 +463,7 @@ } }, "connectVia": { - "referenceName": "PaulsLaptopIR", + "referenceName": "Laptop", "type": "IntegrationRuntimeReference" } }, @@ -477,6 +551,16 @@ }, "dependsOn": [] }, + { + "name": "[concat(parameters('factoryName'), '/default/AzureFunctionTest')]", + "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", + "apiVersion": "2018-06-01", + "properties": { + "privateLinkResourceId": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]", + "groupId": "[parameters('AzureFunctionTest_properties_groupId')]" + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", @@ -548,7 +632,7 @@ "type": "LinkedServiceReference" }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -580,7 +664,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", @@ -600,6 +684,43 @@ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, + { + "name": "[concat(parameters('factoryName'), '/ADLS_PQT_AdventureWorks')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "parameters": { + "EntityName": { + "type": "string" + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@{dataset().EntityName}.parquet", + "type": "Expression" + }, + "folderPath": "Raw", + "fileSystem": "lakeroot" + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" + ] + }, { "name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", @@ -615,7 +736,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "DelimitedText", @@ -638,171 +759,6 @@ "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] - }, - { - "name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]", - "type": "Microsoft.DataFactory/factories/datasets", - "apiVersion": "2018-06-01", - "properties": { - "linkedServiceName": { - "referenceName": "AnyDatabaseConnection", - "type": "LinkedServiceReference", - "parameters": { - "DBConnectionSecret": { - "value": "@dataset().LinkedServiceConnectionSecret", - "type": "Expression" - } - } - }, - "parameters": { - "LinkedServiceConnectionSecret": { - "type": "string" - }, - "SchemaName": { - "type": "string" - }, - "TableName": { - "type": "string" - } - }, - "folder": { - "name": "SQLDB" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [], - "typeProperties": { - "schema": { - "value": "@dataset().SchemaName", - "type": "Expression" - }, - "table": { - "value": "@dataset().TableName", - "type": "Expression" - } - } - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]" - ] - }, - { - "name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]", - "type": "Microsoft.DataFactory/factories/datasets", - "apiVersion": "2018-06-01", - "properties": { - "linkedServiceName": { - "referenceName": "AnyDatabaseConnectionFromAnyKeyVault", - "type": "LinkedServiceReference", - "parameters": { - "KeyVaultURL": { - "value": "@dataset().KeyVaultURL", - "type": "Expression" - }, - "KeyVaultSecretName": { - "value": "@dataset().LinkedServiceConnectionSecret", - "type": "Expression" - } - } - }, - "parameters": { - "LinkedServiceConnectionSecret": { - "type": "string" - }, - "SchemaName": { - "type": "string" - }, - "TableName": { - "type": "string" - }, - "KeyVaultURL": { - "type": "string" - } - }, - "folder": { - "name": "SQLDB" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [], - "typeProperties": { - "schema": { - "value": "@dataset().SchemaName", - "type": "Expression" - }, - "table": { - "value": "@dataset().TableName", - "type": "Expression" - } - } - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]" - ] - }, - { - "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", - "type": "Microsoft.DataFactory/factories/datasets", - "apiVersion": "2018-06-01", - "properties": { - "linkedServiceName": { - "referenceName": "trainingdb01", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "SQLDB" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [], - "typeProperties": {} - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" - ] - }, - { - "name": "[concat(parameters('factoryName'), '/LakeFileParquet')]", - "type": "Microsoft.DataFactory/factories/datasets", - "apiVersion": "2018-06-01", - "properties": { - "linkedServiceName": { - "referenceName": "traininglake01", - "type": "LinkedServiceReference" - }, - "parameters": { - "Directory": { - "type": "string" - }, - "File": { - "type": "string" - } - }, - "folder": { - "name": "Lake" - }, - "annotations": [], - "type": "Parquet", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": { - "value": "@dataset().File", - "type": "Expression" - }, - "folderPath": { - "value": "@dataset().Directory", - "type": "Expression" - }, - "fileSystem": "datawarehouse" - }, - "compressionCodec": "snappy" - }, - "schema": [] - }, - "dependsOn": [ - "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" - ] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json index 889e98c..08cd8d7 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json @@ -12,6 +12,163 @@ "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnection", + "type": "LinkedServiceReference", + "parameters": { + "DBConnectionSecret": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnectionFromAnyKeyVault", + "type": "LinkedServiceReference", + "parameters": { + "KeyVaultURL": { + "value": "@dataset().KeyVaultURL", + "type": "Expression" + }, + "KeyVaultSecretName": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + }, + "KeyVaultURL": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": {} + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/LakeFileParquet')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", "type": "Microsoft.DataFactory/factories/datasets", @@ -1133,11 +1290,13 @@ "lastPublishTime": "2020-09-02T14:13:14Z" }, "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/datasets/LaptopFilesBinary')]", "[concat(variables('factoryId'), '/datasets/LakeFilesBinary')]", "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", "[concat(variables('factoryId'), '/datasets/LakeFiles')]", - "[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]" + "[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]", + "[concat(variables('factoryId'), '/datasets/LakeFileParquet')]" ] }, { @@ -1197,7 +1356,9 @@ "annotations": [], "lastPublishTime": "2020-09-02T14:13:14Z" }, - "dependsOn": [] + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]" + ] }, { "name": "[concat(parameters('factoryName'), '/CopyAWEntity')]", @@ -1272,462 +1433,11 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, "dependsOn": [] - }, - { - "name": "[concat(parameters('factoryName'), '/CopyAWProduct')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "Copy AW Products to data lake", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "0.12:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "BinarySource", - "storeSettings": { - "type": "HttpReadSettings", - "requestMethod": "GET" - }, - "formatSettings": { - "type": "BinaryReadSettings" - } - }, - "sink": { - "type": "BinarySink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - } - }, - "enableStaging": false - }, - "inputs": [ - { - "referenceName": "HTTP_BIN_AWProduct", - "type": "DatasetReference", - "parameters": {} - } - ], - "outputs": [ - { - "referenceName": "ADLS_BIN_AWProduct", - "type": "DatasetReference", - "parameters": {} - } - ] - } - ], - "policy": { - "elapsedTimeMetric": {}, - "cancelAfter": {} - }, - "folder": { - "name": "Labs" - }, - "annotations": [] - }, - "dependsOn": [] - }, - { - "name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "Get Secret", - "type": "WebActivity", - "dependsOn": [ - { - "activity": "Set Key URL", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "url": { - "value": "@variables('CompleteSecretURL')", - "type": "Expression" - }, - "method": "GET", - "headers": {}, - "authentication": { - "type": "MSI", - "resource": "https://vault.azure.net" - } - } - }, - { - "name": "Set Key URL", - "type": "SetVariable", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "variableName": "CompleteSecretURL", - "value": { - "value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')", - "type": "Expression" - } - } - }, - { - "name": "Nested Key Vault URL", - "type": "Lookup", - "dependsOn": [ - { - "activity": "Get Secret", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "dataset": { - "referenceName": "AnyDatabaseTableAnyKeyVault", - "type": "DatasetReference", - "parameters": { - "LinkedServiceConnectionSecret": "ConnectionString-trainingdb01", - "SchemaName": "sys", - "TableName": "objects", - "KeyVaultURL": { - "value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')", - "type": "Expression" - } - } - } - } - } - ], - "policy": { - "elapsedTimeMetric": {}, - "cancelAfter": {} - }, - "parameters": { - "SecretName": { - "type": "string", - "defaultValue": "DemoKeyGetWithWebActivity" - } - }, - "variables": { - "CompleteSecretURL": { - "type": "String" - } - }, - "folder": { - "name": "Demo Pipelines/Misc" - }, - "annotations": [] - }, - "dependsOn": [] - }, - { - "name": "[concat(parameters('factoryName'), '/Lazy Replication')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "Get Table List", - "type": "Lookup", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "sqlReaderQuery": { - "value": "@pipeline().parameters.TableLookupQuery", - "type": "Expression" - }, - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "dataset": { - "referenceName": "AnyDatabaseTable", - "type": "DatasetReference", - "parameters": { - "LinkedServiceConnectionSecret": { - "value": "@pipeline().parameters.SourceConnectionSecret", - "type": "Expression" - }, - "SchemaName": "sys", - "TableName": "objects" - } - }, - "firstRowOnly": false - } - }, - { - "name": "Copy Tables", - "type": "ForEach", - "dependsOn": [ - { - "activity": "Get Table List", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "items": { - "value": "@activity('Get Table List').output.value", - "type": "Expression" - }, - "activities": [ - { - "name": "Copy Table", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "AzureSqlSink", - "preCopyScript": { - "value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}", - "type": "Expression" - }, - "tableOption": "autoCreate", - "disableMetricsCollection": false - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "AnyDatabaseTable", - "type": "DatasetReference", - "parameters": { - "LinkedServiceConnectionSecret": { - "value": "@pipeline().parameters.SourceConnectionSecret", - "type": "Expression" - }, - "SchemaName": { - "value": "@item().SchemaName", - "type": "Expression" - }, - "TableName": { - "value": "@item().TableName", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "AnyDatabaseTable", - "type": "DatasetReference", - "parameters": { - "LinkedServiceConnectionSecret": { - "value": "@pipeline().parameters.TargetConnectionSecret", - "type": "Expression" - }, - "SchemaName": { - "value": "@item().SchemaName", - "type": "Expression" - }, - "TableName": { - "value": "@item().TableName", - "type": "Expression" - } - } - } - ] - } - ] - } - } - ], - "policy": { - "elapsedTimeMetric": {}, - "cancelAfter": {} - }, - "parameters": { - "SourceConnectionSecret": { - "type": "string", - "defaultValue": "ConnectionString-trainingdb01" - }, - "TargetConnectionSecret": { - "type": "string", - "defaultValue": "ConnectionString-trainingdb02" - }, - "TableLookupQuery": { - "type": "string", - "defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'" - } - }, - "folder": { - "name": "Demo Pipelines/Dynamic Linked Services" - }, - "annotations": [], - "lastPublishTime": "2020-08-27T09:43:42Z" - }, - "dependsOn": [] - }, - { - "name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]", - "type": "Microsoft.DataFactory/factories/pipelines", - "apiVersion": "2018-06-01", - "properties": { - "activities": [ - { - "name": "Get Bucket Contents", - "type": "Lookup", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]", - "storedProcedureParameters": { - "BucketId": { - "type": "Int32", - "value": { - "value": "@pipeline().parameters.BucketId", - "type": "Expression" - } - } - }, - "partitionOption": "None" - }, - "dataset": { - "referenceName": "GetSetMetadata", - "type": "DatasetReference", - "parameters": {} - }, - "firstRowOnly": false - } - }, - { - "name": "Execute Processes", - "type": "ForEach", - "dependsOn": [ - { - "activity": "Get Bucket Contents", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "items": { - "value": "@activity('Get Bucket Contents').output.value", - "type": "Expression" - }, - "batchCount": 50, - "activities": [ - { - "name": "Run Process", - "type": "SqlServerStoredProcedure", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "storedProcedureName": { - "value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')", - "type": "Expression" - } - }, - "linkedServiceName": { - "referenceName": "trainingdb01", - "type": "LinkedServiceReference" - } - } - ] - } - } - ], - "policy": { - "elapsedTimeMetric": {}, - "cancelAfter": {} - }, - "parameters": { - "BucketId": { - "type": "int" - } - }, - "folder": { - "name": "Demo Pipelines/Scaling Out" - }, - "annotations": [], - "lastPublishTime": "2020-08-24T13:41:19Z" - }, - "dependsOn": [] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_3.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_3.json index 9414923..d6d330f 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_3.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_3.json @@ -12,6 +12,636 @@ "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ + { + "name": "[concat(parameters('factoryName'), '/CopyAWProduct')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Copy AW Products to data lake", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "HTTP_BIN_AWProduct", + "type": "DatasetReference", + "parameters": {} + } + ], + "outputs": [ + { + "referenceName": "ADLS_BIN_AWProduct", + "type": "DatasetReference", + "parameters": {} + } + ] + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "folder": { + "name": "Labs1" + }, + "annotations": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/CopyAWUsingMetadata')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get catalog", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json", + "method": "GET", + "headers": {} + } + }, + { + "name": "Cache catalog as JSON array", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get catalog", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "Catalog", + "value": { + "value": "@json(activity('Get catalog').output.Response)", + "type": "Expression" + } + } + }, + { + "name": "ForEach AW entity", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Cache catalog as JSON array", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@variables('Catalog')", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy AW entity", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "ParquetWriteSettings" + } + }, + "enableStaging": false, + "translator": { + "value": "@item().translator", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "HTTP_TSV_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "EntityName": { + "value": "@item().EntityName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLS_PQT_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "EntityName": { + "value": "@item().EntityName", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Notify error", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Copy AW entity", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "SendEmail", + "method": "POST", + "headers": {}, + "body": { + "value": "{\n \"emailRecipients\": \"paul@mrpaulandrew.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "EmailSenderFunction", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "variables": { + "Catalog": { + "type": "Array", + "defaultValue": [] + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Secret", + "type": "WebActivity", + "dependsOn": [ + { + "activity": "Set Key URL", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": { + "value": "@variables('CompleteSecretURL')", + "type": "Expression" + }, + "method": "GET", + "headers": {}, + "authentication": { + "type": "MSI", + "resource": "https://vault.azure.net" + } + } + }, + { + "name": "Set Key URL", + "type": "SetVariable", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "variableName": "CompleteSecretURL", + "value": { + "value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')", + "type": "Expression" + } + } + }, + { + "name": "Nested Key Vault URL", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get Secret", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTableAnyKeyVault", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": "ConnectionString-trainingdb01", + "SchemaName": "sys", + "TableName": "objects", + "KeyVaultURL": { + "value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')", + "type": "Expression" + } + } + } + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "SecretName": { + "type": "string", + "defaultValue": "DemoKeyGetWithWebActivity" + } + }, + "variables": { + "CompleteSecretURL": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Lazy Replication')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Table List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TableLookupQuery", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": "sys", + "TableName": "objects" + } + }, + "firstRowOnly": false + } + }, + { + "name": "Copy Tables", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Table List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Table List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy Table", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.TargetConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "SourceConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "TargetConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb02" + }, + "TableLookupQuery": { + "type": "string", + "defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'" + } + }, + "folder": { + "name": "Demo Pipelines/Dynamic Linked Services" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:43:42Z" + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "activities": [ + { + "name": "Get Bucket Contents", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]", + "storedProcedureParameters": { + "BucketId": { + "type": "Int32", + "value": { + "value": "@pipeline().parameters.BucketId", + "type": "Expression" + } + } + }, + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Execute Processes", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Contents", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Contents').output.value", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Run Process", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": { + "value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "policy": { + "elapsedTimeMetric": {}, + "cancelAfter": {} + }, + "parameters": { + "BucketId": { + "type": "int" + } + }, + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "dependsOn": [] + }, { "name": "[concat(parameters('factoryName'), '/MappingOrderAggregation')]", "type": "Microsoft.DataFactory/factories/dataflows", @@ -577,7 +1207,7 @@ "cancelAfter": {} }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, @@ -641,7 +1271,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] }, @@ -946,7 +1576,9 @@ "annotations": [], "lastPublishTime": "2020-08-24T13:41:19Z" }, - "dependsOn": [] + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/Scale Out Level 2')]" + ] } ] } \ No newline at end of file diff --git a/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json b/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json index 95cb04a..7c7854f 100644 --- a/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json +++ b/TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json @@ -107,16 +107,13 @@ "type": "string", "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": { + "AzureFunctionTest_properties_privateLinkResourceId": { "type": "string", - "defaultValue": "adayfullofadf-sql.database.windows.net" + "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions" }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": { + "AzureFunctionTest_properties_groupId": { "type": "string", - "defaultValue": "sql-admin" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": { - "type": "secureString" + "defaultValue": "sites" }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", @@ -200,15 +197,6 @@ }, "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { "value": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": { - "value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": { - "value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]" - }, - "SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": { - "value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]" } } }, @@ -261,6 +249,12 @@ "traininglake01_properties_typeProperties_url": { "value": "[parameters('traininglake01_properties_typeProperties_url')]" }, + "AzureFunctionTest_properties_privateLinkResourceId": { + "value": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]" + }, + "AzureFunctionTest_properties_groupId": { + "value": "[parameters('AzureFunctionTest_properties_groupId')]" + }, "AzureKeyVault1_properties_privateLinkResourceId": { "value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]" },