{ "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "TrainingFactoryDev" }, "ADLS_saintegrationpipelines_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "@linkedService().DBConnectionSecret" }, "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "@linkedService().KeyVaultSecretName" }, "BatchForTraining01_properties_typeProperties_batchUri": { "type": "string", "defaultValue": "https://batchfortraining01.uksouth.batch.azure.com" }, "BatchForTraining01_properties_typeProperties_poolName": { "type": "string", "defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654" }, "BatchForTraining01_properties_typeProperties_accountName": { "type": "string", "defaultValue": "batchfortraining01" }, "LaptopFiles_properties_typeProperties_host": { "type": "string", "defaultValue": "C:\\ADFRoot\\" }, "LaptopFiles_properties_typeProperties_userId": { "type": "string", "defaultValue": "ADFIRUser" }, "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "https://TrainingKeys01.vault.azure.net/" }, "trainingdb01_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "ConnectionString-trainingdb01" }, "traininglake01_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "AzureFunctionTest_properties_privateLinkResourceId": { "type": "string", "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions" }, "AzureFunctionTest_properties_groupId": { "type": "string", "defaultValue": "sites" }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" }, "AzureKeyVault1_properties_groupId": { "type": "string", "defaultValue": "vault" }, "AzureKeyVault1_properties_fqdns": { "type": "array", "defaultValue": [ "trainingkeys01.vaultcore.azure.net" ] } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "HTTP_AWGitHub", "type": "LinkedServiceReference" }, "parameters": { "Entity": { "type": "string" } }, "folder": { "name": "Labs1" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "HttpServerLocation", "relativeUrl": { "value": "@{dataset().Entity}.csv", "type": "Expression" } } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/HTTP_TSV_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "HTTP_AWGitHub", "type": "LinkedServiceReference" }, "parameters": { "EntityName": { "type": "string" } }, "folder": { "name": "Labs2" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "HttpServerLocation", "relativeUrl": { "value": "@{dataset().EntityName}.csv", "type": "Expression" } }, "columnDelimiter": "\t", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01noneKV", "type": "LinkedServiceReference" }, "folder": { "name": "Lake" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "OrderDetailLines.parquet", "folderPath": "Raw", "fileSystem": "datawarehouse" }, "compressionCodec": "snappy" }, "schema": [ { "name": "SalesOrderID", "type": "INT32" }, { "name": "SalesOrderDetailID", "type": "INT32" }, { "name": "OrderQty", "type": "INT32" }, { "name": "ProductID", "type": "INT32" }, { "name": "UnitPrice", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "UnitPriceDiscount", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "LineTotal", "type": "DECIMAL", "precision": 38, "scale": 6 }, { "name": "rowguid", "type": "UTF8" }, { "name": "ModifiedDate", "type": "INT96" } ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderHeaderParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01noneKV", "type": "LinkedServiceReference" }, "folder": { "name": "Lake" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "OrderHeader.parquet", "folderPath": "Raw", "fileSystem": "datawarehouse" }, "compressionCodec": "snappy" }, "schema": [ { "name": "SalesOrderID", "type": "INT32" }, { "name": "RevisionNumber", "type": "INT32" }, { "name": "OrderDate", "type": "INT96" }, { "name": "DueDate", "type": "INT96" }, { "name": "ShipDate", "type": "INT96" }, { "name": "Status", "type": "INT32" }, { "name": "OnlineOrderFlag", "type": "BOOLEAN" }, { "name": "SalesOrderNumber", "type": "UTF8" }, { "name": "PurchaseOrderNumber", "type": "UTF8" }, { "name": "AccountNumber", "type": "UTF8" }, { "name": "CustomerID", "type": "INT32" }, { "name": "ShipToAddressID", "type": "INT32" }, { "name": "BillToAddressID", "type": "INT32" }, { "name": "ShipMethod", "type": "UTF8" }, { "name": "CreditCardApprovalCode", "type": "UTF8" }, { "name": "SubTotal", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "TaxAmt", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "Freight", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "TotalDue", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "Comment", "type": "UTF8" }, { "name": "rowguid", "type": "UTF8" }, { "name": "ModifiedDate", "type": "INT96" } ] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/ADLS_saintegrationpipelines')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('ADLS_saintegrationpipelines_properties_typeProperties_url')]", "accountKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "traininglake01" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "parameters": { "DBConnectionSecret": { "type": "string" } }, "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": { "value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]", "type": "Expression" } } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "parameters": { "KeyVaultURL": { "type": "string" }, "KeyVaultSecretName": { "type": "string" } }, "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "GenericKeys", "type": "LinkedServiceReference", "parameters": { "baseUrl": { "value": "@linkedService().KeyVaultURL", "type": "Expression" } } }, "secretName": { "value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]", "type": "Expression" } } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/BatchForTraining01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBatch", "typeProperties": { "batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]", "poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]", "accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]", "linkedServiceName": { "referenceName": "TrainingStore01", "type": "LinkedServiceReference" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/LaptopFiles')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "FileServer", "typeProperties": { "host": "[parameters('LaptopFiles_properties_typeProperties_host')]", "userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]", "password": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "ADFIRUser" } }, "connectVia": { "referenceName": "Laptop", "type": "IntegrationRuntimeReference" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]", "credential": { "referenceName": "TrainingCredential01", "type": "CredentialReference" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/trainingdb01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/traininglake01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('traininglake01_properties_typeProperties_url')]", "accountKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "traininglake01" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", "apiVersion": "2018-06-01", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "UK South", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 0, "cleanup": true } } }, "managedVirtualNetwork": { "type": "ManagedVirtualNetworkReference", "referenceName": "default" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/default/AzureFunctionTest')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", "apiVersion": "2018-06-01", "properties": { "privateLinkResourceId": "[parameters('AzureFunctionTest_properties_privateLinkResourceId')]", "groupId": "[parameters('AzureFunctionTest_properties_groupId')]" }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", "apiVersion": "2018-06-01", "properties": { "privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]", "groupId": "[parameters('AzureKeyVault1_properties_groupId')]", "fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]" }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Custom Activity')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "HelloWorld", "type": "Custom", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "command": "CustomActivity.exe", "resourceLinkedService": { "referenceName": "TrainingStore01", "type": "LinkedServiceReference" }, "folderPath": "customactivities", "referenceObjects": { "linkedServices": [], "datasets": [] } }, "linkedServiceName": { "referenceName": "BatchForTraining01", "type": "LinkedServiceReference" } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Misc" }, "annotations": [], "lastPublishTime": "2022-02-03T14:39:04Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/BatchForTraining01')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_BIN_AWProduct')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "folder": { "name": "Labs1" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "Product.csv", "folderPath": "Raw", "fileSystem": "lakeroot" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_BIN_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "parameters": { "Entity": { "type": "string" } }, "folder": { "name": "Labs1" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@{dataset().Entity}.tsv", "type": "Expression" }, "folderPath": "Raw", "fileSystem": "lakeroot" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_PQT_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "parameters": { "EntityName": { "type": "string" } }, "folder": { "name": "Labs2" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@{dataset().EntityName}.parquet", "type": "Expression" }, "folderPath": "Raw", "fileSystem": "lakeroot" }, "compressionCodec": "snappy" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] }, { "name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "parameters": { "FileName": { "type": "string" } }, "folder": { "name": "Labs1" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().FileName", "type": "Expression" }, "folderPath": "Raw", "fileSystem": "lakeroot" }, "columnDelimiter": "\t", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]" ] } ] }