ARM template and parameters deployed on 4-22-2023 16:56:6, based on the collaboration branch's commit ID: 5955c39d4c8acc1d92007beeeda942da0f7894e0
This commit is contained in:
Paul Andrew
parent
07d95ecd8a
commit
62020408ab
@ -107,16 +107,13 @@
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"AzureFunctionTest_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
"defaultValue": "adayfullofadf-sql.database.windows.net"
|
||||
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"AzureFunctionTest_properties_groupId": {
|
||||
"type": "string",
|
||||
"defaultValue": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"type": "secureString"
|
||||
"defaultValue": "sites"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
@ -1480,7 +1477,7 @@
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
@ -1544,7 +1541,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
@ -1625,7 +1622,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
@ -1692,7 +1689,7 @@
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
@ -1701,6 +1698,189 @@
|
||||
"[concat(variables('factoryId'), '/datasets/ADLS_BIN_AWProduct')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyAWUsingMetadata')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get catalog",
|
||||
"type": "WebActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json",
|
||||
"method": "GET",
|
||||
"headers": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Cache catalog as JSON array",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get catalog",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "Catalog",
|
||||
"value": {
|
||||
"value": "@json(activity('Get catalog').output.Response)",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ForEach AW entity",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Cache catalog as JSON array",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@variables('Catalog')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy AW entity",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "DelimitedTextSource",
|
||||
"storeSettings": {
|
||||
"type": "HttpReadSettings",
|
||||
"requestMethod": "GET"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "ParquetWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"translator": {
|
||||
"value": "@item().translator",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "HTTP_TSV_AdventureWorks",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"value": "@item().EntityName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLS_PQT_AdventureWorks",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"value": "@item().EntityName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Notify error",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy AW entity",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "SendEmail",
|
||||
"method": "POST",
|
||||
"headers": {},
|
||||
"body": {
|
||||
"value": "{\n \"emailRecipients\": \"paul@mrpaulandrew.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "EmailSenderFunction",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"variables": {
|
||||
"Catalog": {
|
||||
"type": "Array",
|
||||
"defaultValue": []
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs2"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/HTTP_TSV_AdventureWorks')]",
|
||||
"[concat(variables('factoryId'), '/datasets/ADLS_PQT_AdventureWorks')]",
|
||||
"[concat(variables('factoryId'), '/linkedServices/EmailSenderFunction')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Custom Activity')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
@ -2549,7 +2729,7 @@
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -2581,7 +2761,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -2601,6 +2781,43 @@
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_PQT_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "ADLS_saintegrationpipelines",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs2"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@{dataset().EntityName}.parquet",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": "Raw",
|
||||
"fileSystem": "lakeroot"
|
||||
},
|
||||
"compressionCodec": "snappy"
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
@ -2616,7 +2833,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
@ -2772,7 +2989,7 @@
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -2801,7 +3018,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -2819,6 +3036,43 @@
|
||||
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_TSV_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "HTTP_AWGitHub",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs2"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "HttpServerLocation",
|
||||
"relativeUrl": {
|
||||
"value": "@{dataset().EntityName}.csv",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"columnDelimiter": "\t",
|
||||
"escapeChar": "\\",
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
@ -3650,12 +3904,12 @@
|
||||
}
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "PaulsLaptopIR",
|
||||
"referenceName": "Laptop",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]",
|
||||
"[concat(variables('factoryId'), '/integrationRuntimes/Laptop')]",
|
||||
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
|
||||
]
|
||||
},
|
||||
@ -3783,6 +4037,26 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/trigger1')]",
|
||||
"type": "Microsoft.DataFactory/factories/triggers",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"runtimeState": "Stopped",
|
||||
"pipelines": [],
|
||||
"type": "ScheduleTrigger",
|
||||
"typeProperties": {
|
||||
"recurrence": {
|
||||
"frequency": "Minute",
|
||||
"interval": 15,
|
||||
"startTime": "2023-02-06T14:56:00Z",
|
||||
"timeZone": "UTC"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
@ -3817,36 +4091,6 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/SSISIntegrationRuntime')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"type": "Managed",
|
||||
"typeProperties": {
|
||||
"computeProperties": {
|
||||
"location": "UK South",
|
||||
"nodeSize": "Standard_D2_v3",
|
||||
"numberOfNodes": 1,
|
||||
"maxParallelExecutionsPerNode": 2
|
||||
},
|
||||
"ssisProperties": {
|
||||
"catalogInfo": {
|
||||
"catalogServerEndpoint": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]",
|
||||
"catalogAdminUserName": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]",
|
||||
"catalogAdminPassword": {
|
||||
"type": "SecureString",
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]"
|
||||
},
|
||||
"catalogPricingTier": "S1"
|
||||
},
|
||||
"edition": "Standard",
|
||||
"licenseType": "LicenseIncluded"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
@ -4363,6 +4607,18 @@
|
||||
"properties": {},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/default/AzureFunctionTest')]",
|
||||
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"privateLinkResourceId": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]",
|
||||
"groupId": "[parameters('AzureFunctionTest_properties_groupId')]"
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]",
|
||||
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
|
||||
@ -4375,6 +4631,16 @@
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Laptop')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"type": "SelfHosted",
|
||||
"typeProperties": {}
|
||||
},
|
||||
"dependsOn": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -80,14 +80,11 @@
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"value": "adayfullofadf-sql.database.windows.net"
|
||||
"AzureFunctionTest_properties_privateLinkResourceId": {
|
||||
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"value": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"value": ""
|
||||
"AzureFunctionTest_properties_groupId": {
|
||||
"value": "sites"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
||||
|
||||
@ -80,14 +80,11 @@
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"value": "adayfullofadf-sql.database.windows.net"
|
||||
"AzureFunctionTest_properties_privateLinkResourceId": {
|
||||
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"value": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"value": ""
|
||||
"AzureFunctionTest_properties_groupId": {
|
||||
"value": "sites"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
||||
|
||||
@ -62,17 +62,6 @@
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"type": "string",
|
||||
"defaultValue": "adayfullofadf-sql.database.windows.net"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"type": "string",
|
||||
"defaultValue": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"type": "secureString"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
@ -644,6 +633,26 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/trigger1')]",
|
||||
"type": "Microsoft.DataFactory/factories/triggers",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"runtimeState": "Stopped",
|
||||
"pipelines": [],
|
||||
"type": "ScheduleTrigger",
|
||||
"typeProperties": {
|
||||
"recurrence": {
|
||||
"frequency": "Minute",
|
||||
"interval": 15,
|
||||
"startTime": "2023-02-06T14:56:00Z",
|
||||
"timeZone": "UTC"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
@ -678,36 +687,6 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/SSISIntegrationRuntime')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"type": "Managed",
|
||||
"typeProperties": {
|
||||
"computeProperties": {
|
||||
"location": "UK South",
|
||||
"nodeSize": "Standard_D2_v3",
|
||||
"numberOfNodes": 1,
|
||||
"maxParallelExecutionsPerNode": 2
|
||||
},
|
||||
"ssisProperties": {
|
||||
"catalogInfo": {
|
||||
"catalogServerEndpoint": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]",
|
||||
"catalogAdminUserName": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]",
|
||||
"catalogAdminPassword": {
|
||||
"type": "SecureString",
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]"
|
||||
},
|
||||
"catalogPricingTier": "S1"
|
||||
},
|
||||
"edition": "Standard",
|
||||
"licenseType": "LicenseIncluded"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/TrainingCredential01')]",
|
||||
"type": "Microsoft.DataFactory/factories/credentials",
|
||||
@ -727,6 +706,16 @@
|
||||
"properties": {},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Laptop')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"type": "SelfHosted",
|
||||
"typeProperties": {}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/EmailSender')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
@ -798,7 +787,7 @@
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -811,39 +800,6 @@
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWProduct')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "HTTP_AWGitHub",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Entity": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "HttpServerLocation",
|
||||
"relativeUrl": {
|
||||
"value": "@{dataset().Entity}.csv",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -51,6 +51,14 @@
|
||||
"type": "string",
|
||||
"defaultValue": "https://traininglake01.dfs.core.windows.net"
|
||||
},
|
||||
"AzureFunctionTest_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions"
|
||||
},
|
||||
"AzureFunctionTest_properties_groupId": {
|
||||
"type": "string",
|
||||
"defaultValue": "sites"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
||||
@ -70,6 +78,72 @@
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "HTTP_AWGitHub",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Entity": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "HttpServerLocation",
|
||||
"relativeUrl": {
|
||||
"value": "@{dataset().Entity}.csv",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_TSV_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "HTTP_AWGitHub",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs2"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "HttpServerLocation",
|
||||
"relativeUrl": {
|
||||
"value": "@{dataset().EntityName}.csv",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"columnDelimiter": "\t",
|
||||
"escapeChar": "\\",
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
@ -389,7 +463,7 @@
|
||||
}
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "PaulsLaptopIR",
|
||||
"referenceName": "Laptop",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
},
|
||||
@ -477,6 +551,16 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/default/AzureFunctionTest')]",
|
||||
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"privateLinkResourceId": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]",
|
||||
"groupId": "[parameters('AzureFunctionTest_properties_groupId')]"
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]",
|
||||
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
|
||||
@ -548,7 +632,7 @@
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -580,7 +664,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
@ -600,6 +684,43 @@
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_PQT_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "ADLS_saintegrationpipelines",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs2"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@{dataset().EntityName}.parquet",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": "Raw",
|
||||
"fileSystem": "lakeroot"
|
||||
},
|
||||
"compressionCodec": "snappy"
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
@ -615,7 +736,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
@ -638,171 +759,6 @@
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AnyDatabaseConnection",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"DBConnectionSecret": {
|
||||
"value": "@dataset().LinkedServiceConnectionSecret",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"type": "string"
|
||||
},
|
||||
"SchemaName": {
|
||||
"type": "string"
|
||||
},
|
||||
"TableName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [],
|
||||
"typeProperties": {
|
||||
"schema": {
|
||||
"value": "@dataset().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"table": {
|
||||
"value": "@dataset().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"KeyVaultURL": {
|
||||
"value": "@dataset().KeyVaultURL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultSecretName": {
|
||||
"value": "@dataset().LinkedServiceConnectionSecret",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"type": "string"
|
||||
},
|
||||
"SchemaName": {
|
||||
"type": "string"
|
||||
},
|
||||
"TableName": {
|
||||
"type": "string"
|
||||
},
|
||||
"KeyVaultURL": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [],
|
||||
"typeProperties": {
|
||||
"schema": {
|
||||
"value": "@dataset().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"table": {
|
||||
"value": "@dataset().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [],
|
||||
"typeProperties": {}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFileParquet')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "traininglake01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"File": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Lake"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
},
|
||||
"fileSystem": "datawarehouse"
|
||||
},
|
||||
"compressionCodec": "snappy"
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -12,6 +12,163 @@
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AnyDatabaseConnection",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"DBConnectionSecret": {
|
||||
"value": "@dataset().LinkedServiceConnectionSecret",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"type": "string"
|
||||
},
|
||||
"SchemaName": {
|
||||
"type": "string"
|
||||
},
|
||||
"TableName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [],
|
||||
"typeProperties": {
|
||||
"schema": {
|
||||
"value": "@dataset().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"table": {
|
||||
"value": "@dataset().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"KeyVaultURL": {
|
||||
"value": "@dataset().KeyVaultURL",
|
||||
"type": "Expression"
|
||||
},
|
||||
"KeyVaultSecretName": {
|
||||
"value": "@dataset().LinkedServiceConnectionSecret",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"type": "string"
|
||||
},
|
||||
"SchemaName": {
|
||||
"type": "string"
|
||||
},
|
||||
"TableName": {
|
||||
"type": "string"
|
||||
},
|
||||
"KeyVaultURL": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [],
|
||||
"typeProperties": {
|
||||
"schema": {
|
||||
"value": "@dataset().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"table": {
|
||||
"value": "@dataset().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [],
|
||||
"typeProperties": {}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFileParquet')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "traininglake01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"File": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Lake"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
},
|
||||
"fileSystem": "datawarehouse"
|
||||
},
|
||||
"compressionCodec": "snappy"
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
@ -1133,11 +1290,13 @@
|
||||
"lastPublishTime": "2020-09-02T14:13:14Z"
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LaptopFilesBinary')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LakeFilesBinary')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LakeFiles')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]"
|
||||
"[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LakeFileParquet')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -1197,7 +1356,9 @@
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-09-02T14:13:14Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyAWEntity')]",
|
||||
@ -1272,462 +1433,11 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyAWProduct')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy AW Products to data lake",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "HttpReadSettings",
|
||||
"requestMethod": "GET"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "HTTP_BIN_AWProduct",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLS_BIN_AWProduct",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Secret",
|
||||
"type": "WebActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Set Key URL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"url": {
|
||||
"value": "@variables('CompleteSecretURL')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"authentication": {
|
||||
"type": "MSI",
|
||||
"resource": "https://vault.azure.net"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Set Key URL",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "CompleteSecretURL",
|
||||
"value": {
|
||||
"value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Nested Key Vault URL",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Secret",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AnyDatabaseTableAnyKeyVault",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": "ConnectionString-trainingdb01",
|
||||
"SchemaName": "sys",
|
||||
"TableName": "objects",
|
||||
"KeyVaultURL": {
|
||||
"value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"parameters": {
|
||||
"SecretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "DemoKeyGetWithWebActivity"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"CompleteSecretURL": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Misc"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Lazy Replication')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Table List",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@pipeline().parameters.TableLookupQuery",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AnyDatabaseTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"value": "@pipeline().parameters.SourceConnectionSecret",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SchemaName": "sys",
|
||||
"TableName": "objects"
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Copy Tables",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Table List",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('Get Table List').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Table",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"tableOption": "autoCreate",
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"translator": {
|
||||
"type": "TabularTranslator",
|
||||
"typeConversion": true,
|
||||
"typeConversionSettings": {
|
||||
"allowDataTruncation": true,
|
||||
"treatBooleanAsNumber": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AnyDatabaseTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"value": "@pipeline().parameters.SourceConnectionSecret",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SchemaName": {
|
||||
"value": "@item().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"value": "@item().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AnyDatabaseTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"value": "@pipeline().parameters.TargetConnectionSecret",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SchemaName": {
|
||||
"value": "@item().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"value": "@item().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"parameters": {
|
||||
"SourceConnectionSecret": {
|
||||
"type": "string",
|
||||
"defaultValue": "ConnectionString-trainingdb01"
|
||||
},
|
||||
"TargetConnectionSecret": {
|
||||
"type": "string",
|
||||
"defaultValue": "ConnectionString-trainingdb02"
|
||||
},
|
||||
"TableLookupQuery": {
|
||||
"type": "string",
|
||||
"defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Dynamic Linked Services"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-27T09:43:42Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Bucket Contents",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]",
|
||||
"storedProcedureParameters": {
|
||||
"BucketId": {
|
||||
"type": "Int32",
|
||||
"value": {
|
||||
"value": "@pipeline().parameters.BucketId",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "GetSetMetadata",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute Processes",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Bucket Contents",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('Get Bucket Contents').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"batchCount": 50,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Run Process",
|
||||
"type": "SqlServerStoredProcedure",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"storedProcedureName": {
|
||||
"value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"parameters": {
|
||||
"BucketId": {
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Scaling Out"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-24T13:41:19Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -12,6 +12,636 @@
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyAWProduct')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy AW Products to data lake",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "BinarySource",
|
||||
"storeSettings": {
|
||||
"type": "HttpReadSettings",
|
||||
"requestMethod": "GET"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "BinaryReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "BinarySink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "HTTP_BIN_AWProduct",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLS_BIN_AWProduct",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyAWUsingMetadata')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get catalog",
|
||||
"type": "WebActivity",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json",
|
||||
"method": "GET",
|
||||
"headers": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Cache catalog as JSON array",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get catalog",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "Catalog",
|
||||
"value": {
|
||||
"value": "@json(activity('Get catalog').output.Response)",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "ForEach AW entity",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Cache catalog as JSON array",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@variables('Catalog')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy AW entity",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "DelimitedTextSource",
|
||||
"storeSettings": {
|
||||
"type": "HttpReadSettings",
|
||||
"requestMethod": "GET"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "ParquetSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "ParquetWriteSettings"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"translator": {
|
||||
"value": "@item().translator",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "HTTP_TSV_AdventureWorks",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"value": "@item().EntityName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "ADLS_PQT_AdventureWorks",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"EntityName": {
|
||||
"value": "@item().EntityName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Notify error",
|
||||
"type": "AzureFunctionActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Copy AW entity",
|
||||
"dependencyConditions": [
|
||||
"Failed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"functionName": "SendEmail",
|
||||
"method": "POST",
|
||||
"headers": {},
|
||||
"body": {
|
||||
"value": "{\n \"emailRecipients\": \"paul@mrpaulandrew.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "EmailSenderFunction",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"variables": {
|
||||
"Catalog": {
|
||||
"type": "Array",
|
||||
"defaultValue": []
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs2"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Secret",
|
||||
"type": "WebActivity",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Set Key URL",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"url": {
|
||||
"value": "@variables('CompleteSecretURL')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"method": "GET",
|
||||
"headers": {},
|
||||
"authentication": {
|
||||
"type": "MSI",
|
||||
"resource": "https://vault.azure.net"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Set Key URL",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "CompleteSecretURL",
|
||||
"value": {
|
||||
"value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Nested Key Vault URL",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Secret",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AnyDatabaseTableAnyKeyVault",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": "ConnectionString-trainingdb01",
|
||||
"SchemaName": "sys",
|
||||
"TableName": "objects",
|
||||
"KeyVaultURL": {
|
||||
"value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"parameters": {
|
||||
"SecretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "DemoKeyGetWithWebActivity"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"CompleteSecretURL": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Misc"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Lazy Replication')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Table List",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderQuery": {
|
||||
"value": "@pipeline().parameters.TableLookupQuery",
|
||||
"type": "Expression"
|
||||
},
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "AnyDatabaseTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"value": "@pipeline().parameters.SourceConnectionSecret",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SchemaName": "sys",
|
||||
"TableName": "objects"
|
||||
}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Copy Tables",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Table List",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('Get Table List').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy Table",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"queryTimeout": "02:00:00",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"sink": {
|
||||
"type": "AzureSqlSink",
|
||||
"preCopyScript": {
|
||||
"value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}",
|
||||
"type": "Expression"
|
||||
},
|
||||
"tableOption": "autoCreate",
|
||||
"disableMetricsCollection": false
|
||||
},
|
||||
"enableStaging": false,
|
||||
"translator": {
|
||||
"type": "TabularTranslator",
|
||||
"typeConversion": true,
|
||||
"typeConversionSettings": {
|
||||
"allowDataTruncation": true,
|
||||
"treatBooleanAsNumber": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "AnyDatabaseTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"value": "@pipeline().parameters.SourceConnectionSecret",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SchemaName": {
|
||||
"value": "@item().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"value": "@item().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "AnyDatabaseTable",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"LinkedServiceConnectionSecret": {
|
||||
"value": "@pipeline().parameters.TargetConnectionSecret",
|
||||
"type": "Expression"
|
||||
},
|
||||
"SchemaName": {
|
||||
"value": "@item().SchemaName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"TableName": {
|
||||
"value": "@item().TableName",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"parameters": {
|
||||
"SourceConnectionSecret": {
|
||||
"type": "string",
|
||||
"defaultValue": "ConnectionString-trainingdb01"
|
||||
},
|
||||
"TargetConnectionSecret": {
|
||||
"type": "string",
|
||||
"defaultValue": "ConnectionString-trainingdb02"
|
||||
},
|
||||
"TableLookupQuery": {
|
||||
"type": "string",
|
||||
"defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Dynamic Linked Services"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-27T09:43:42Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Bucket Contents",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]",
|
||||
"storedProcedureParameters": {
|
||||
"BucketId": {
|
||||
"type": "Int32",
|
||||
"value": {
|
||||
"value": "@pipeline().parameters.BucketId",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "GetSetMetadata",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Execute Processes",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Bucket Contents",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('Get Bucket Contents').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"batchCount": 50,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Run Process",
|
||||
"type": "SqlServerStoredProcedure",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"storedProcedureName": {
|
||||
"value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"parameters": {
|
||||
"BucketId": {
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Scaling Out"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-24T13:41:19Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/MappingOrderAggregation')]",
|
||||
"type": "Microsoft.DataFactory/factories/dataflows",
|
||||
@ -577,7 +1207,7 @@
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
@ -641,7 +1271,7 @@
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
"name": "Labs1"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
@ -946,7 +1576,9 @@
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-24T13:41:19Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/pipelines/Scale Out Level 2')]"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -107,16 +107,13 @@
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"AzureFunctionTest_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
"defaultValue": "adayfullofadf-sql.database.windows.net"
|
||||
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"AzureFunctionTest_properties_groupId": {
|
||||
"type": "string",
|
||||
"defaultValue": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"type": "secureString"
|
||||
"defaultValue": "sites"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
@ -200,15 +197,6 @@
|
||||
},
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"value": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
@ -261,6 +249,12 @@
|
||||
"traininglake01_properties_typeProperties_url": {
|
||||
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
|
||||
},
|
||||
"AzureFunctionTest_properties_privateLinkResourceId": {
|
||||
"value": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]"
|
||||
},
|
||||
"AzureFunctionTest_properties_groupId": {
|
||||
"value": "[parameters('AzureFunctionTest_properties_groupId')]"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]"
|
||||
},
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user