ARM template and parameters deployed on 8-30-2022 11:50:9, based on the collaboration branch's commit ID: 08c36321ad6faf7da0993a00cb843c42d9b981c7
This commit is contained in:
Paul Andrew
parent
2a23b67be2
commit
38224d36dc
File diff suppressed because it is too large
Load Diff
@ -5,6 +5,9 @@
|
||||
"factoryName": {
|
||||
"value": "TrainingFactoryDev"
|
||||
},
|
||||
"AzureSqlDatabase1_connectionString": {
|
||||
"value": ""
|
||||
},
|
||||
"TrainingStore01_connectionString": {
|
||||
"value": ""
|
||||
},
|
||||
@ -14,12 +17,18 @@
|
||||
"traininglake01noneKV_accountKey": {
|
||||
"value": ""
|
||||
},
|
||||
"ADLS_saintegrationpipelines_properties_typeProperties_url": {
|
||||
"value": "https://traininglake01.dfs.core.windows.net"
|
||||
},
|
||||
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
|
||||
"value": "@linkedService().DBConnectionSecret"
|
||||
},
|
||||
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
|
||||
"value": "@linkedService().KeyVaultSecretName"
|
||||
},
|
||||
"AzureDataLakeStorage1_properties_typeProperties_url": {
|
||||
"value": "https://adayfullofadfsa.dfs.core.windows.net"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_batchUri": {
|
||||
"value": "https://batchfortraining01.uksouth.batch.azure.com"
|
||||
},
|
||||
@ -32,6 +41,12 @@
|
||||
"GenericKeys_properties_typeProperties_baseUrl": {
|
||||
"value": "@{linkedService().baseUrl}"
|
||||
},
|
||||
"HTTP_AWGitHub_properties_typeProperties_url": {
|
||||
"value": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/"
|
||||
},
|
||||
"HTTP_AWProduct_properties_typeProperties_url": {
|
||||
"value": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_host": {
|
||||
"value": "C:\\ADFRoot\\"
|
||||
},
|
||||
@ -59,6 +74,15 @@
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"value": "adayfullofadf-sql.database.windows.net"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"value": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"value": ""
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
||||
},
|
||||
|
||||
@ -5,6 +5,9 @@
|
||||
"factoryName": {
|
||||
"value": "TrainingFactoryDev"
|
||||
},
|
||||
"AzureSqlDatabase1_connectionString": {
|
||||
"value": ""
|
||||
},
|
||||
"TrainingStore01_connectionString": {
|
||||
"value": ""
|
||||
},
|
||||
@ -14,12 +17,18 @@
|
||||
"traininglake01noneKV_accountKey": {
|
||||
"value": ""
|
||||
},
|
||||
"ADLS_saintegrationpipelines_properties_typeProperties_url": {
|
||||
"value": "https://traininglake01.dfs.core.windows.net"
|
||||
},
|
||||
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
|
||||
"value": "@linkedService().DBConnectionSecret"
|
||||
},
|
||||
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
|
||||
"value": "@linkedService().KeyVaultSecretName"
|
||||
},
|
||||
"AzureDataLakeStorage1_properties_typeProperties_url": {
|
||||
"value": "https://adayfullofadfsa.dfs.core.windows.net"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_batchUri": {
|
||||
"value": "https://batchfortraining01.uksouth.batch.azure.com"
|
||||
},
|
||||
@ -32,6 +41,12 @@
|
||||
"GenericKeys_properties_typeProperties_baseUrl": {
|
||||
"value": "@{linkedService().baseUrl}"
|
||||
},
|
||||
"HTTP_AWGitHub_properties_typeProperties_url": {
|
||||
"value": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/"
|
||||
},
|
||||
"HTTP_AWProduct_properties_typeProperties_url": {
|
||||
"value": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_host": {
|
||||
"value": "C:\\ADFRoot\\"
|
||||
},
|
||||
@ -59,6 +74,15 @@
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"value": "adayfullofadf-sql.database.windows.net"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"value": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"value": ""
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
||||
},
|
||||
|
||||
@ -7,6 +7,10 @@
|
||||
"metadata": "Data Factory name",
|
||||
"defaultValue": "TrainingFactoryDev"
|
||||
},
|
||||
"AzureSqlDatabase1_connectionString": {
|
||||
"type": "secureString",
|
||||
"metadata": "Secure string for 'connectionString' of 'AzureSqlDatabase1'"
|
||||
},
|
||||
"TrainingStore01_connectionString": {
|
||||
"type": "secureString",
|
||||
"metadata": "Secure string for 'connectionString' of 'TrainingStore01'"
|
||||
@ -19,10 +23,22 @@
|
||||
"type": "secureString",
|
||||
"metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'"
|
||||
},
|
||||
"AzureDataLakeStorage1_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://adayfullofadfsa.dfs.core.windows.net"
|
||||
},
|
||||
"GenericKeys_properties_typeProperties_baseUrl": {
|
||||
"type": "string",
|
||||
"defaultValue": "@{linkedService().baseUrl}"
|
||||
},
|
||||
"HTTP_AWGitHub_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/"
|
||||
},
|
||||
"HTTP_AWProduct_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv"
|
||||
},
|
||||
"TrainingKeys01_properties_typeProperties_baseUrl": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
|
||||
@ -39,41 +55,16 @@
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"type": "string",
|
||||
"defaultValue": "@linkedService().DBConnectionSecret"
|
||||
"defaultValue": "adayfullofadf-sql.database.windows.net"
|
||||
},
|
||||
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"type": "string",
|
||||
"defaultValue": "@linkedService().KeyVaultSecretName"
|
||||
"defaultValue": "sql-admin"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_batchUri": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://batchfortraining01.uksouth.batch.azure.com"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_poolName": {
|
||||
"type": "string",
|
||||
"defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_accountName": {
|
||||
"type": "string",
|
||||
"defaultValue": "batchfortraining01"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_host": {
|
||||
"type": "string",
|
||||
"defaultValue": "C:\\ADFRoot\\"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_userId": {
|
||||
"type": "string",
|
||||
"defaultValue": "ADFIRUser"
|
||||
},
|
||||
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
|
||||
},
|
||||
"trainingdb01_properties_typeProperties_connectionString_secretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "ConnectionString-trainingdb01"
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"type": "secureString"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
@ -492,6 +483,32 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureDataLakeStorage1')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureBlobFS",
|
||||
"typeProperties": {
|
||||
"url": "[parameters('AzureDataLakeStorage1_properties_typeProperties_url')]"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AzureSqlDatabase1')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": "[parameters('AzureSqlDatabase1_connectionString')]"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/GenericKeys')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
@ -511,6 +528,36 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_AWGitHub')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "HttpServer",
|
||||
"typeProperties": {
|
||||
"url": "[parameters('HTTP_AWGitHub_properties_typeProperties_url')]",
|
||||
"enableServerCertificateValidation": true,
|
||||
"authenticationType": "Anonymous"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_AWProduct')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "HttpServer",
|
||||
"typeProperties": {
|
||||
"url": "[parameters('HTTP_AWProduct_properties_typeProperties_url')]",
|
||||
"enableServerCertificateValidation": true,
|
||||
"authenticationType": "Anonymous"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/TrainingKeys01')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
@ -605,6 +652,36 @@
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/SSISIntegrationRuntime')]",
|
||||
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"type": "Managed",
|
||||
"typeProperties": {
|
||||
"computeProperties": {
|
||||
"location": "UK South",
|
||||
"nodeSize": "Standard_D2_v3",
|
||||
"numberOfNodes": 1,
|
||||
"maxParallelExecutionsPerNode": 2
|
||||
},
|
||||
"ssisProperties": {
|
||||
"catalogInfo": {
|
||||
"catalogServerEndpoint": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]",
|
||||
"catalogAdminUserName": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]",
|
||||
"catalogAdminPassword": {
|
||||
"type": "SecureString",
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]"
|
||||
},
|
||||
"catalogPricingTier": "S1"
|
||||
},
|
||||
"edition": "Standard",
|
||||
"licenseType": "LicenseIncluded"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/TrainingCredential01')]",
|
||||
"type": "Microsoft.DataFactory/factories/credentials",
|
||||
@ -625,28 +702,61 @@
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/pipeline1')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_BIN_AWProduct')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Wait1",
|
||||
"type": "Wait",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"waitTimeInSeconds": 1
|
||||
"linkedServiceName": {
|
||||
"referenceName": "HTTP_AWProduct",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "HttpServerLocation"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWProduct')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "HTTP_AWGitHub",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Entity": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "HttpServerLocation",
|
||||
"relativeUrl": {
|
||||
"value": "@{dataset().Entity}.csv",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"annotations": []
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]",
|
||||
@ -845,166 +955,6 @@
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
"DBConnectionSecret": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": {
|
||||
"value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
"KeyVaultURL": {
|
||||
"type": "string"
|
||||
},
|
||||
"KeyVaultSecretName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "GenericKeys",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"baseUrl": {
|
||||
"value": "@linkedService().KeyVaultURL",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"secretName": {
|
||||
"value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/GenericKeys')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/BatchForTraining01')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureBatch",
|
||||
"typeProperties": {
|
||||
"batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]",
|
||||
"poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]",
|
||||
"accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "TrainingStore01",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopFiles')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "FileServer",
|
||||
"typeProperties": {
|
||||
"host": "[parameters('LaptopFiles_properties_typeProperties_host')]",
|
||||
"userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]",
|
||||
"password": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": "ADFIRUser"
|
||||
}
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "PaulsLaptopIR",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]",
|
||||
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureKeyVault",
|
||||
"typeProperties": {
|
||||
"baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]",
|
||||
"credential": {
|
||||
"referenceName": "TrainingCredential01",
|
||||
"type": "CredentialReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/credentials/TrainingCredential01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/trainingdb01')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -7,6 +7,46 @@
|
||||
"metadata": "Data Factory name",
|
||||
"defaultValue": "TrainingFactoryDev"
|
||||
},
|
||||
"ADLS_saintegrationpipelines_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://traininglake01.dfs.core.windows.net"
|
||||
},
|
||||
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "@linkedService().DBConnectionSecret"
|
||||
},
|
||||
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "@linkedService().KeyVaultSecretName"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_batchUri": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://batchfortraining01.uksouth.batch.azure.com"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_poolName": {
|
||||
"type": "string",
|
||||
"defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_accountName": {
|
||||
"type": "string",
|
||||
"defaultValue": "batchfortraining01"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_host": {
|
||||
"type": "string",
|
||||
"defaultValue": "C:\\ADFRoot\\"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_userId": {
|
||||
"type": "string",
|
||||
"defaultValue": "ADFIRUser"
|
||||
},
|
||||
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
|
||||
},
|
||||
"trainingdb01_properties_typeProperties_connectionString_secretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "ConnectionString-trainingdb01"
|
||||
},
|
||||
"traininglake01_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://traininglake01.dfs.core.windows.net"
|
||||
@ -30,6 +70,174 @@
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_saintegrationpipelines')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureBlobFS",
|
||||
"typeProperties": {
|
||||
"url": "[parameters('ADLS_saintegrationpipelines_properties_typeProperties_url')]",
|
||||
"accountKey": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": "traininglake01"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
"DBConnectionSecret": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": {
|
||||
"value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"parameters": {
|
||||
"KeyVaultURL": {
|
||||
"type": "string"
|
||||
},
|
||||
"KeyVaultSecretName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "GenericKeys",
|
||||
"type": "LinkedServiceReference",
|
||||
"parameters": {
|
||||
"baseUrl": {
|
||||
"value": "@linkedService().KeyVaultURL",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
"secretName": {
|
||||
"value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/BatchForTraining01')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureBatch",
|
||||
"typeProperties": {
|
||||
"batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]",
|
||||
"poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]",
|
||||
"accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]",
|
||||
"linkedServiceName": {
|
||||
"referenceName": "TrainingStore01",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopFiles')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "FileServer",
|
||||
"typeProperties": {
|
||||
"host": "[parameters('LaptopFiles_properties_typeProperties_host')]",
|
||||
"userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]",
|
||||
"password": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": "ADFIRUser"
|
||||
}
|
||||
},
|
||||
"connectVia": {
|
||||
"referenceName": "PaulsLaptopIR",
|
||||
"type": "IntegrationRuntimeReference"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureKeyVault",
|
||||
"typeProperties": {
|
||||
"baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]",
|
||||
"credential": {
|
||||
"referenceName": "TrainingCredential01",
|
||||
"type": "CredentialReference"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/trainingdb01')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureSqlDatabase",
|
||||
"typeProperties": {
|
||||
"connectionString": {
|
||||
"type": "AzureKeyVaultSecret",
|
||||
"store": {
|
||||
"referenceName": "TrainingKeys01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/traininglake01')]",
|
||||
"type": "Microsoft.DataFactory/factories/linkedServices",
|
||||
@ -132,7 +340,110 @@
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2022-02-03T14:39:04Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/BatchForTraining01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_BIN_AWProduct')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "ADLS_saintegrationpipelines",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": "Product.csv",
|
||||
"folderPath": "Raw",
|
||||
"fileSystem": "lakeroot"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_BIN_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "ADLS_saintegrationpipelines",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Entity": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@{dataset().Entity}.tsv",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": "Raw",
|
||||
"fileSystem": "lakeroot"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "ADLS_saintegrationpipelines",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"FileName": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().FileName",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": "Raw",
|
||||
"fileSystem": "lakeroot"
|
||||
},
|
||||
"columnDelimiter": "\t",
|
||||
"escapeChar": "\\",
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]",
|
||||
@ -177,7 +488,9 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]",
|
||||
@ -229,7 +542,9 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
|
||||
@ -248,7 +563,9 @@
|
||||
"schema": [],
|
||||
"typeProperties": {}
|
||||
},
|
||||
"dependsOn": []
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFileParquet')]",
|
||||
@ -378,463 +695,6 @@
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "traininglake01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"File": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Lake"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "AzureBlobFSLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
},
|
||||
"fileSystem": "datawarehouse"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "LaptopFiles",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "Laptop"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "FileServerLocation",
|
||||
"fileName": "Person.csv",
|
||||
"folderPath": "ForUpload/People"
|
||||
},
|
||||
"columnDelimiter": ",",
|
||||
"escapeChar": "\\",
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": [
|
||||
{
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"type": "String"
|
||||
}
|
||||
]
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "LaptopFiles",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"File": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Laptop"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Binary",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "FileServerLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "LaptopFiles",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"File": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Laptop"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "Parquet",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "FileServerLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"compressionCodec": "snappy"
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopFolders')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "LaptopFiles",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Laptop"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "FileServerLocation",
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"columnDelimiter": ",",
|
||||
"escapeChar": "\\",
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/LaptopsFiles')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "LaptopFiles",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
"Directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"File": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Laptop"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "DelimitedText",
|
||||
"typeProperties": {
|
||||
"location": {
|
||||
"type": "FileServerLocation",
|
||||
"fileName": {
|
||||
"value": "@dataset().File",
|
||||
"type": "Expression"
|
||||
},
|
||||
"folderPath": {
|
||||
"value": "@dataset().Directory",
|
||||
"type": "Expression"
|
||||
}
|
||||
},
|
||||
"columnDelimiter": ",",
|
||||
"escapeChar": "\\",
|
||||
"quoteChar": "\""
|
||||
},
|
||||
"schema": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/TableOrderSummary')]",
|
||||
"type": "Microsoft.DataFactory/factories/datasets",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"folder": {
|
||||
"name": "SQLDB"
|
||||
},
|
||||
"annotations": [],
|
||||
"type": "AzureSqlTable",
|
||||
"schema": [
|
||||
{
|
||||
"name": "SalesOrderNumber",
|
||||
"type": "varchar"
|
||||
},
|
||||
{
|
||||
"name": "RecordCount",
|
||||
"type": "int",
|
||||
"precision": 10
|
||||
}
|
||||
],
|
||||
"typeProperties": {
|
||||
"schema": "dbo",
|
||||
"table": "OrderSummary"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]",
|
||||
"type": "Microsoft.DataFactory/factories/dataflows",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"type": "WranglingDataFlow",
|
||||
"typeProperties": {
|
||||
"sources": [
|
||||
{
|
||||
"name": "LakeFileOrderDetailLinesParquet",
|
||||
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
|
||||
"dataset": {
|
||||
"referenceName": "LakeFileOrderDetailLinesParquet",
|
||||
"type": "DatasetReference"
|
||||
}
|
||||
}
|
||||
],
|
||||
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Upload Person",
|
||||
"description": "hfdhgfdhfd",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "DelimitedTextSource",
|
||||
"storeSettings": {
|
||||
"type": "FileServerReadSettings",
|
||||
"recursive": true
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "DelimitedTextSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextWriteSettings",
|
||||
"quoteAllText": true,
|
||||
"fileExtension": ".txt"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"translator": {
|
||||
"type": "TabularTranslator",
|
||||
"typeConversion": true,
|
||||
"typeConversionSettings": {
|
||||
"allowDataTruncation": true,
|
||||
"treatBooleanAsNumber": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "LaptopFilePersonCSV",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "LakeFilePersonCSV",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Data Uploads"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-09-29T13:50:28Z"
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Upload Any File",
|
||||
"type": "Copy",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "DelimitedTextSource",
|
||||
"storeSettings": {
|
||||
"type": "FileServerReadSettings",
|
||||
"recursive": true
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextReadSettings"
|
||||
}
|
||||
},
|
||||
"sink": {
|
||||
"type": "DelimitedTextSink",
|
||||
"storeSettings": {
|
||||
"type": "AzureBlobFSWriteSettings"
|
||||
},
|
||||
"formatSettings": {
|
||||
"type": "DelimitedTextWriteSettings",
|
||||
"quoteAllText": true,
|
||||
"fileExtension": ".txt"
|
||||
}
|
||||
},
|
||||
"enableStaging": false,
|
||||
"translator": {
|
||||
"type": "TabularTranslator",
|
||||
"typeConversion": true,
|
||||
"typeConversionSettings": {
|
||||
"allowDataTruncation": true,
|
||||
"treatBooleanAsNumber": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"referenceName": "LaptopsFiles",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Directory": "ForUpload\\People",
|
||||
"File": "Person.csv"
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"referenceName": "LakeFiles",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {
|
||||
"Directory": "Landing",
|
||||
"File": "Person.csv"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Data Uploads"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-27T09:06:46Z"
|
||||
},
|
||||
"dependsOn": [
|
||||
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
|
||||
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
625
TrainingFactoryDev/linkedTemplates/ArmTemplate_3.json
Normal file
625
TrainingFactoryDev/linkedTemplates/ArmTemplate_3.json
Normal file
@ -0,0 +1,625 @@
|
||||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"type": "string",
|
||||
"metadata": "Data Factory name",
|
||||
"defaultValue": "TrainingFactoryDev"
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/UpdateProductDimension_Sorted')]",
|
||||
"type": "Microsoft.DataFactory/factories/dataflows",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"type": "MappingDataFlow",
|
||||
"typeProperties": {
|
||||
"sources": [
|
||||
{
|
||||
"dataset": {
|
||||
"referenceName": "ADLS_TSV_AdventureWorks",
|
||||
"type": "DatasetReference"
|
||||
},
|
||||
"name": "Product"
|
||||
},
|
||||
{
|
||||
"dataset": {
|
||||
"referenceName": "ADLS_TSV_AdventureWorks",
|
||||
"type": "DatasetReference"
|
||||
},
|
||||
"name": "ProductSubcategory"
|
||||
},
|
||||
{
|
||||
"dataset": {
|
||||
"referenceName": "ADLS_TSV_AdventureWorks",
|
||||
"type": "DatasetReference"
|
||||
},
|
||||
"name": "ProductCategory"
|
||||
}
|
||||
],
|
||||
"sinks": [
|
||||
{
|
||||
"linkedService": {
|
||||
"referenceName": "ADLS_saintegrationpipelines",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"name": "WriteToDataLake"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"name": "SelectProductColumns"
|
||||
},
|
||||
{
|
||||
"name": "SelectSubcategoryColumns"
|
||||
},
|
||||
{
|
||||
"name": "SelectCategoryColumns"
|
||||
},
|
||||
{
|
||||
"name": "LookupProductCategory"
|
||||
},
|
||||
{
|
||||
"name": "LookupProductSubcategory"
|
||||
},
|
||||
{
|
||||
"name": "RemoveDuplicateColumns"
|
||||
},
|
||||
{
|
||||
"name": "SortBySubcategory"
|
||||
}
|
||||
],
|
||||
"scriptLines": [
|
||||
"source(output(",
|
||||
" ProductId as integer,",
|
||||
" Product as string,",
|
||||
" {_col2_} as string,",
|
||||
" {_col3_} as boolean,",
|
||||
" {_col4_} as boolean,",
|
||||
" {_col5_} as string,",
|
||||
" {_col6_} as short,",
|
||||
" {_col7_} as short,",
|
||||
" {_col8_} as double,",
|
||||
" {_col9_} as double,",
|
||||
" {_col10_} as string,",
|
||||
" {_col11_} as string,",
|
||||
" {_col12_} as string,",
|
||||
" {_col13_} as double,",
|
||||
" {_col14_} as short,",
|
||||
" {_col15_} as string,",
|
||||
" {_col16_} as string,",
|
||||
" {_col17_} as string,",
|
||||
" SubcategoryId as integer,",
|
||||
" {_col19_} as short,",
|
||||
" {_col20_} as timestamp,",
|
||||
" {_col21_} as timestamp,",
|
||||
" {_col22_} as string,",
|
||||
" {_col23_} as string,",
|
||||
" {_col24_} as string",
|
||||
" ),",
|
||||
" allowSchemaDrift: true,",
|
||||
" validateSchema: false,",
|
||||
" ignoreNoFilesFound: false) ~> Product",
|
||||
"source(output(",
|
||||
" SubcategoryId as integer,",
|
||||
" CategoryId as integer,",
|
||||
" Subcategory as string,",
|
||||
" {_col3_} as string,",
|
||||
" {_col4_} as timestamp",
|
||||
" ),",
|
||||
" allowSchemaDrift: true,",
|
||||
" validateSchema: false,",
|
||||
" ignoreNoFilesFound: false) ~> ProductSubcategory",
|
||||
"source(output(",
|
||||
" CategoryId as integer,",
|
||||
" Category as string,",
|
||||
" {_col2_} as string,",
|
||||
" {_col3_} as timestamp",
|
||||
" ),",
|
||||
" allowSchemaDrift: true,",
|
||||
" validateSchema: false,",
|
||||
" ignoreNoFilesFound: false) ~> ProductCategory",
|
||||
"Product select(mapColumn(",
|
||||
" ProductId,",
|
||||
" Product,",
|
||||
" SubcategoryId",
|
||||
" ),",
|
||||
" skipDuplicateMapInputs: true,",
|
||||
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
|
||||
"ProductSubcategory select(mapColumn(",
|
||||
" SubcategoryId,",
|
||||
" CategoryId,",
|
||||
" Subcategory",
|
||||
" ),",
|
||||
" skipDuplicateMapInputs: true,",
|
||||
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
|
||||
"ProductCategory select(mapColumn(",
|
||||
" CategoryId,",
|
||||
" Category",
|
||||
" ),",
|
||||
" skipDuplicateMapInputs: true,",
|
||||
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
|
||||
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
|
||||
" multiple: false,",
|
||||
" pickup: 'any',",
|
||||
" broadcast: 'auto')~> LookupProductCategory",
|
||||
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
|
||||
" multiple: false,",
|
||||
" pickup: 'any',",
|
||||
" broadcast: 'auto')~> LookupProductSubcategory",
|
||||
"LookupProductSubcategory select(mapColumn(",
|
||||
" ProductId,",
|
||||
" Product,",
|
||||
" SubcategoryId = SelectProductColumns@SubcategoryId,",
|
||||
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
|
||||
" CategoryId = SelectSubcategoryColumns@CategoryId,",
|
||||
" Subcategory,",
|
||||
" CategoryId = SelectCategoryColumns@CategoryId,",
|
||||
" Category",
|
||||
" ),",
|
||||
" skipDuplicateMapInputs: true,",
|
||||
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
|
||||
"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
|
||||
"SortBySubcategory sink(allowSchemaDrift: true,",
|
||||
" validateSchema: false,",
|
||||
" format: 'parquet',",
|
||||
" fileSystem: 'lakeroot',",
|
||||
" folderPath: 'Conformed/DimProduct',",
|
||||
" truncate: true,",
|
||||
" umask: 0022,",
|
||||
" preCommands: [],",
|
||||
" postCommands: [],",
|
||||
" skipDuplicateMapInputs: true,",
|
||||
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
|
||||
]
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/06 - Example Global Param')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "06 - Get Any File List",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "05 - Get File List Utility",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": false,
|
||||
"parameters": {
|
||||
"GetFileQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Set Pipe Output",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "06 - Get Any File List",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "WorkerOutput",
|
||||
"value": {
|
||||
"value": "@activity('06 - Get Any File List').output.pipelineRunId",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Set Global Param",
|
||||
"type": "SetVariable",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "06 - Get Any File List",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"variableName": "GlobalParam",
|
||||
"value": {
|
||||
"value": "@concat(pipeline().globalParameters.Environment,'Test')",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"variables": {
|
||||
"WorkerOutput": {
|
||||
"type": "String"
|
||||
},
|
||||
"GlobalParam": {
|
||||
"type": "String"
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Data Uploads"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/BuildDimProduct')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Run UpdateProductDimension",
|
||||
"type": "ExecuteDataFlow",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "0.12:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataflow": {
|
||||
"referenceName": "UpdateProductDimension",
|
||||
"type": "DataFlowReference",
|
||||
"parameters": {},
|
||||
"datasetParameters": {
|
||||
"Product": {
|
||||
"FileName": "Product.tsv"
|
||||
},
|
||||
"ProductSubcategory": {
|
||||
"FileName": "ProductSubcategory.tsv"
|
||||
},
|
||||
"ProductCategory": {
|
||||
"FileName": "ProductCategory.tsv"
|
||||
},
|
||||
"WriteToDataLake": {}
|
||||
}
|
||||
},
|
||||
"staging": {},
|
||||
"compute": {
|
||||
"coreCount": 8,
|
||||
"computeType": "General"
|
||||
},
|
||||
"traceLevel": "Fine"
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/CopyAWEntities')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Copy each AW entity",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@variables('Entities')",
|
||||
"type": "Expression"
|
||||
},
|
||||
"isSequential": false,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Execute CopyAWEntity",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "CopyAWEntity",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"Entity": {
|
||||
"value": "@item()",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"variables": {
|
||||
"Entities": {
|
||||
"type": "Array",
|
||||
"defaultValue": [
|
||||
"Product",
|
||||
"ProductSubcategory",
|
||||
"ProductCategory"
|
||||
]
|
||||
}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Labs"
|
||||
},
|
||||
"annotations": []
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Order Summary with Mapping')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Mapping Order Aggregation",
|
||||
"type": "ExecuteDataFlow",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"dataflow": {
|
||||
"referenceName": "MappingOrderAggregation",
|
||||
"type": "DataFlowReference",
|
||||
"parameters": {},
|
||||
"datasetParameters": {
|
||||
"OrderHeader": {},
|
||||
"OrderLineDetails": {},
|
||||
"OrderSummary": {}
|
||||
}
|
||||
},
|
||||
"staging": {},
|
||||
"compute": {
|
||||
"coreCount": 8,
|
||||
"computeType": "General"
|
||||
},
|
||||
"traceLevel": "Fine"
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Data Flows"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-09-02T14:13:15Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "[concat(parameters('factoryName'), '/Scale Out Level 1')]",
|
||||
"type": "Microsoft.DataFactory/factories/pipelines",
|
||||
"apiVersion": "2018-06-01",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
"name": "Get Bucket Ids",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Reset And Log Start",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
},
|
||||
{
|
||||
"activity": "Reset Bucket Process Map",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderStoredProcedureName": "[[dbo].[GetBuckets]",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "GetSetMetadata",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
},
|
||||
"firstRowOnly": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Call Buckets",
|
||||
"type": "ForEach",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Get Bucket Ids",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
}
|
||||
],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"items": {
|
||||
"value": "@activity('Get Bucket Ids').output.value",
|
||||
"type": "Expression"
|
||||
},
|
||||
"isSequential": false,
|
||||
"batchCount": 50,
|
||||
"activities": [
|
||||
{
|
||||
"name": "Call Level 2",
|
||||
"type": "ExecutePipeline",
|
||||
"dependsOn": [],
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Scale Out Level 2",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
"parameters": {
|
||||
"BucketId": {
|
||||
"value": "@{item().BucketId}",
|
||||
"type": "Expression"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Log End",
|
||||
"type": "SqlServerStoredProcedure",
|
||||
"dependsOn": [
|
||||
{
|
||||
"activity": "Call Buckets",
|
||||
"dependencyConditions": [
|
||||
"Succeeded"
|
||||
]
|
||||
},
|
||||
{
|
||||
"activity": "Reset And Log Start",
|
||||
"dependencyConditions": [
|
||||
"Completed"
|
||||
]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"storedProcedureName": "[[dbo].[SetBucketLogEntry]",
|
||||
"storedProcedureParameters": {
|
||||
"LogId": {
|
||||
"value": {
|
||||
"value": "@activity('Reset And Log Start').output.firstRow.LogId",
|
||||
"type": "Expression"
|
||||
},
|
||||
"type": "Int32"
|
||||
}
|
||||
}
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Reset And Log Start",
|
||||
"type": "Lookup",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"source": {
|
||||
"type": "AzureSqlSource",
|
||||
"sqlReaderStoredProcedureName": "[[dbo].[SetBucketLogEntry]",
|
||||
"partitionOption": "None"
|
||||
},
|
||||
"dataset": {
|
||||
"referenceName": "GetSetMetadata",
|
||||
"type": "DatasetReference",
|
||||
"parameters": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Reset Bucket Process Map",
|
||||
"type": "SqlServerStoredProcedure",
|
||||
"dependsOn": [],
|
||||
"policy": {
|
||||
"timeout": "7.00:00:00",
|
||||
"retry": 0,
|
||||
"retryIntervalInSeconds": 30,
|
||||
"secureOutput": false,
|
||||
"secureInput": false
|
||||
},
|
||||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"storedProcedureName": "[[dbo].[SetBucketProcesses]"
|
||||
},
|
||||
"linkedServiceName": {
|
||||
"referenceName": "trainingdb01",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"elapsedTimeMetric": {},
|
||||
"cancelAfter": {}
|
||||
},
|
||||
"folder": {
|
||||
"name": "Demo Pipelines/Scaling Out"
|
||||
},
|
||||
"annotations": [],
|
||||
"lastPublishTime": "2020-08-24T13:41:19Z"
|
||||
},
|
||||
"dependsOn": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -7,6 +7,10 @@
|
||||
"metadata": "Data Factory name",
|
||||
"defaultValue": "TrainingFactoryDev"
|
||||
},
|
||||
"AzureSqlDatabase1_connectionString": {
|
||||
"type": "secureString",
|
||||
"metadata": "Secure string for 'connectionString' of 'AzureSqlDatabase1'"
|
||||
},
|
||||
"TrainingStore01_connectionString": {
|
||||
"type": "secureString",
|
||||
"metadata": "Secure string for 'connectionString' of 'TrainingStore01'"
|
||||
@ -19,6 +23,10 @@
|
||||
"type": "secureString",
|
||||
"metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'"
|
||||
},
|
||||
"ADLS_saintegrationpipelines_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://traininglake01.dfs.core.windows.net"
|
||||
},
|
||||
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
|
||||
"type": "string",
|
||||
"defaultValue": "@linkedService().DBConnectionSecret"
|
||||
@ -27,6 +35,10 @@
|
||||
"type": "string",
|
||||
"defaultValue": "@linkedService().KeyVaultSecretName"
|
||||
},
|
||||
"AzureDataLakeStorage1_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://adayfullofadfsa.dfs.core.windows.net"
|
||||
},
|
||||
"BatchForTraining01_properties_typeProperties_batchUri": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://batchfortraining01.uksouth.batch.azure.com"
|
||||
@ -43,6 +55,14 @@
|
||||
"type": "string",
|
||||
"defaultValue": "@{linkedService().baseUrl}"
|
||||
},
|
||||
"HTTP_AWGitHub_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/"
|
||||
},
|
||||
"HTTP_AWProduct_properties_typeProperties_url": {
|
||||
"type": "string",
|
||||
"defaultValue": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv"
|
||||
},
|
||||
"LaptopFiles_properties_typeProperties_host": {
|
||||
"type": "string",
|
||||
"defaultValue": "C:\\ADFRoot\\"
|
||||
@ -79,6 +99,17 @@
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"type": "string",
|
||||
"defaultValue": "adayfullofadf-sql.database.windows.net"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"type": "string",
|
||||
"defaultValue": "sql-admin"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"type": "secureString"
|
||||
},
|
||||
"AzureKeyVault1_properties_privateLinkResourceId": {
|
||||
"type": "string",
|
||||
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
||||
@ -120,6 +151,9 @@
|
||||
"factoryName": {
|
||||
"value": "[parameters('factoryName')]"
|
||||
},
|
||||
"AzureSqlDatabase1_connectionString": {
|
||||
"value": "[parameters('AzureSqlDatabase1_connectionString')]"
|
||||
},
|
||||
"TrainingStore01_connectionString": {
|
||||
"value": "[parameters('TrainingStore01_connectionString')]"
|
||||
},
|
||||
@ -129,9 +163,18 @@
|
||||
"traininglake01noneKV_accountKey": {
|
||||
"value": "[parameters('traininglake01noneKV_accountKey')]"
|
||||
},
|
||||
"AzureDataLakeStorage1_properties_typeProperties_url": {
|
||||
"value": "[parameters('AzureDataLakeStorage1_properties_typeProperties_url')]"
|
||||
},
|
||||
"GenericKeys_properties_typeProperties_baseUrl": {
|
||||
"value": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]"
|
||||
},
|
||||
"HTTP_AWGitHub_properties_typeProperties_url": {
|
||||
"value": "[parameters('HTTP_AWGitHub_properties_typeProperties_url')]"
|
||||
},
|
||||
"HTTP_AWProduct_properties_typeProperties_url": {
|
||||
"value": "[parameters('HTTP_AWProduct_properties_typeProperties_url')]"
|
||||
},
|
||||
"TrainingKeys01_properties_typeProperties_baseUrl": {
|
||||
"value": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]"
|
||||
},
|
||||
@ -144,6 +187,36 @@
|
||||
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
|
||||
"value": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]"
|
||||
},
|
||||
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
|
||||
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "TrainingFactoryDev_ArmTemplate_1",
|
||||
"type": "Microsoft.Resources/deployments",
|
||||
"apiVersion": "2018-02-01",
|
||||
"properties": {
|
||||
"mode": "incremental",
|
||||
"templateLink": {
|
||||
"uri": "[concat(parameters('containerUri'), '/ArmTemplate_1.json', parameters('containerSasToken'))]",
|
||||
"contentVersion": "1.0.0.0"
|
||||
},
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"value": "[parameters('factoryName')]"
|
||||
},
|
||||
"ADLS_saintegrationpipelines_properties_typeProperties_url": {
|
||||
"value": "[parameters('ADLS_saintegrationpipelines_properties_typeProperties_url')]"
|
||||
},
|
||||
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
|
||||
"value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]"
|
||||
},
|
||||
@ -170,24 +243,6 @@
|
||||
},
|
||||
"trainingdb01_properties_typeProperties_connectionString_secretName": {
|
||||
"value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"name": "TrainingFactoryDev_ArmTemplate_1",
|
||||
"type": "Microsoft.Resources/deployments",
|
||||
"apiVersion": "2018-02-01",
|
||||
"properties": {
|
||||
"mode": "incremental",
|
||||
"templateLink": {
|
||||
"uri": "[concat(parameters('containerUri'), '/ArmTemplate_1.json', parameters('containerSasToken'))]",
|
||||
"contentVersion": "1.0.0.0"
|
||||
},
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"value": "[parameters('factoryName')]"
|
||||
},
|
||||
"traininglake01_properties_typeProperties_url": {
|
||||
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
|
||||
@ -226,6 +281,26 @@
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Resources/deployments', 'TrainingFactoryDev_ArmTemplate_1')]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "TrainingFactoryDev_ArmTemplate_3",
|
||||
"type": "Microsoft.Resources/deployments",
|
||||
"apiVersion": "2018-02-01",
|
||||
"properties": {
|
||||
"mode": "incremental",
|
||||
"templateLink": {
|
||||
"uri": "[concat(parameters('containerUri'), '/ArmTemplate_3.json', parameters('containerSasToken'))]",
|
||||
"contentVersion": "1.0.0.0"
|
||||
},
|
||||
"parameters": {
|
||||
"factoryName": {
|
||||
"value": "[parameters('factoryName')]"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Resources/deployments', 'TrainingFactoryDev_ArmTemplate_2')]"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user