{ "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "TrainingFactoryDev" }, "ADLS_saintegrationpipelines_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "@linkedService().DBConnectionSecret" }, "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "@linkedService().KeyVaultSecretName" }, "BatchForTraining01_properties_typeProperties_batchUri": { "type": "string", "defaultValue": "https://batchfortraining01.uksouth.batch.azure.com" }, "BatchForTraining01_properties_typeProperties_poolName": { "type": "string", "defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654" }, "BatchForTraining01_properties_typeProperties_accountName": { "type": "string", "defaultValue": "batchfortraining01" }, "LaptopFiles_properties_typeProperties_host": { "type": "string", "defaultValue": "C:\\ADFRoot\\" }, "LaptopFiles_properties_typeProperties_userId": { "type": "string", "defaultValue": "ADFIRUser" }, "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "https://TrainingKeys01.vault.azure.net/" }, "trainingdb01_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "ConnectionString-trainingdb01" }, "traininglake01_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" }, "AzureKeyVault1_properties_groupId": { "type": "string", "defaultValue": "vault" }, "AzureKeyVault1_properties_fqdns": { "type": "array", "defaultValue": [ "trainingkeys01.vaultcore.azure.net" ] } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/ADLS_saintegrationpipelines')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('ADLS_saintegrationpipelines_properties_typeProperties_url')]", "accountKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "traininglake01" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "parameters": { "DBConnectionSecret": { "type": "string" } }, "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": { "value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]", "type": "Expression" } } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "parameters": { "KeyVaultURL": { "type": "string" }, "KeyVaultSecretName": { "type": "string" } }, "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "GenericKeys", "type": "LinkedServiceReference", "parameters": { "baseUrl": { "value": "@linkedService().KeyVaultURL", "type": "Expression" } } }, "secretName": { "value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]", "type": "Expression" } } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/BatchForTraining01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBatch", "typeProperties": { "batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]", "poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]", "accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]", "linkedServiceName": { "referenceName": "TrainingStore01", "type": "LinkedServiceReference" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/LaptopFiles')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "FileServer", "typeProperties": { "host": "[parameters('LaptopFiles_properties_typeProperties_host')]", "userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]", "password": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "ADFIRUser" } }, "connectVia": { "referenceName": "PaulsLaptopIR", "type": "IntegrationRuntimeReference" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]", "credential": { "referenceName": "TrainingCredential01", "type": "CredentialReference" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/trainingdb01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/traininglake01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('traininglake01_properties_typeProperties_url')]", "accountKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "traininglake01" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", "apiVersion": "2018-06-01", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "UK South", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 0, "cleanup": true } } }, "managedVirtualNetwork": { "type": "ManagedVirtualNetworkReference", "referenceName": "default" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", "apiVersion": "2018-06-01", "properties": { "privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]", "groupId": "[parameters('AzureKeyVault1_properties_groupId')]", "fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]" }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Custom Activity')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "HelloWorld", "type": "Custom", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "command": "CustomActivity.exe", "resourceLinkedService": { "referenceName": "TrainingStore01", "type": "LinkedServiceReference" }, "folderPath": "customactivities", "referenceObjects": { "linkedServices": [], "datasets": [] } }, "linkedServiceName": { "referenceName": "BatchForTraining01", "type": "LinkedServiceReference" } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Misc" }, "annotations": [], "lastPublishTime": "2022-02-03T14:39:04Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/BatchForTraining01')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_BIN_AWProduct')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "folder": { "name": "Labs" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "Product.csv", "folderPath": "Raw", "fileSystem": "lakeroot" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_BIN_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "parameters": { "Entity": { "type": "string" } }, "folder": { "name": "Labs" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@{dataset().Entity}.tsv", "type": "Expression" }, "folderPath": "Raw", "fileSystem": "lakeroot" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "parameters": { "FileName": { "type": "string" } }, "folder": { "name": "Labs" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().FileName", "type": "Expression" }, "folderPath": "Raw", "fileSystem": "lakeroot" }, "columnDelimiter": "\t", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "AnyDatabaseConnection", "type": "LinkedServiceReference", "parameters": { "DBConnectionSecret": { "value": "@dataset().LinkedServiceConnectionSecret", "type": "Expression" } } }, "parameters": { "LinkedServiceConnectionSecret": { "type": "string" }, "SchemaName": { "type": "string" }, "TableName": { "type": "string" } }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": { "schema": { "value": "@dataset().SchemaName", "type": "Expression" }, "table": { "value": "@dataset().TableName", "type": "Expression" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]" ] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "AnyDatabaseConnectionFromAnyKeyVault", "type": "LinkedServiceReference", "parameters": { "KeyVaultURL": { "value": "@dataset().KeyVaultURL", "type": "Expression" }, "KeyVaultSecretName": { "value": "@dataset().LinkedServiceConnectionSecret", "type": "Expression" } } }, "parameters": { "LinkedServiceConnectionSecret": { "type": "string" }, "SchemaName": { "type": "string" }, "TableName": { "type": "string" }, "KeyVaultURL": { "type": "string" } }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": { "schema": { "value": "@dataset().SchemaName", "type": "Expression" }, "table": { "value": "@dataset().TableName", "type": "Expression" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]" ] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFileParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Lake" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" }, "fileSystem": "datawarehouse" }, "compressionCodec": "snappy" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "folder": { "name": "Lake" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "Person.csv", "folderPath": "Landing", "fileSystem": "datawarehouse" }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [ { "type": "String" }, { "type": "String" }, { "type": "String" } ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFiles')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Lake" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" }, "fileSystem": "datawarehouse" }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] } ] }