dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Azure-Data-Integration-Pipe.../TrainingFactoryDev/ARMTemplateForFactory.json

4179 lines
111 KiB
JSON
Raw Blame History

{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name",
"defaultValue": "TrainingFactoryDev"
},
"AzureSqlDatabase1_connectionString": {
"type": "secureString",
"metadata": "Secure string for 'connectionString' of 'AzureSqlDatabase1'"
},
"TrainingStore01_connectionString": {
"type": "secureString",
"metadata": "Secure string for 'connectionString' of 'TrainingStore01'"
},
"traininglak01_accountKey": {
"type": "secureString",
"metadata": "Secure string for 'accountKey' of 'traininglak01'"
},
"traininglake01noneKV_accountKey": {
"type": "secureString",
"metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'"
},
"ADLS_saintegrationpipelines_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "@linkedService().DBConnectionSecret"
},
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "@linkedService().KeyVaultSecretName"
},
"AzureDataLakeStorage1_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://adayfullofadfsa.dfs.core.windows.net"
},
"BatchForTraining01_properties_typeProperties_batchUri": {
"type": "string",
"defaultValue": "https://batchfortraining01.uksouth.batch.azure.com"
},
"BatchForTraining01_properties_typeProperties_poolName": {
"type": "string",
"defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654"
},
"BatchForTraining01_properties_typeProperties_accountName": {
"type": "string",
"defaultValue": "batchfortraining01"
},
"GenericKeys_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": "@{linkedService().baseUrl}"
},
"HTTP_AWGitHub_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/"
},
"HTTP_AWProduct_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv"
},
"LaptopFiles_properties_typeProperties_host": {
"type": "string",
"defaultValue": "C:\\ADFRoot\\"
},
"LaptopFiles_properties_typeProperties_userId": {
"type": "string",
"defaultValue": "ADFIRUser"
},
"TrainingKeys01_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
},
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
},
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb01"
},
"traininglak01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01noneKV_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
"type": "string",
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
},
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint": {
"type": "string",
"defaultValue": "adayfullofadf-sql.database.windows.net"
},
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName": {
"type": "string",
"defaultValue": "sql-admin"
},
"SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword": {
"type": "secureString"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"type": "string",
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
},
"AzureKeyVault1_properties_groupId": {
"type": "string",
"defaultValue": "vault"
},
"AzureKeyVault1_properties_fqdns": {
"type": "array",
"defaultValue": [
"trainingkeys01.vaultcore.azure.net"
]
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Upload Person",
"description": "hfdhgfdhfd",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopFilePersonCSV",
"type": "DatasetReference",
"parameters": {}
}
],
"outputs": [
{
"referenceName": "LakeFilePersonCSV",
"type": "DatasetReference",
"parameters": {}
}
]
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-09-29T13:50:28Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]",
"[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Upload Any File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": "Person.csv"
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": "Person.csv"
}
}
]
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:46Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopFolders')]",
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "UploadFiles",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Log Upload Start",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Copy By File Type",
"type": "Switch",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"on": {
"value": "@substring(item().FileName,add(indexof(item().FileName,'.'),1),sub(length(item().FileName),add(indexof(item().FileName,'.'),1)))",
"type": "Expression"
},
"cases": [
{
"value": "csv",
"activities": [
{
"name": "CSV Copy",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"additionalColumns": [
{
"name": "FilePath",
"value": "$$FILEPATH"
},
{
"name": "Environment",
"value": {
"value": "@pipeline().globalParameters.Environment",
"type": "Expression"
}
},
{
"name": "PipelineName",
"value": {
"value": "@pipeline().Pipeline",
"type": "Expression"
}
},
{
"name": "RunId",
"value": {
"value": "@pipeline().RunId",
"type": "Expression"
}
},
{
"name": "RunDate",
"value": {
"value": "@utcnow()",
"type": "Expression"
}
}
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": false,
"enablePartitionDiscovery": false
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().SourceDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().TargetDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
]
}
]
},
{
"value": "parquet",
"activities": [
{
"name": "Parquet Copy",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "ParquetSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": false,
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "ParquetSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "ParquetWriteSettings"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopFilesParquet",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().SourceDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFileParquet",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().TargetDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
]
}
]
}
],
"defaultActivities": [
{
"name": "Binary Copy",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": false
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "LaptopFilesBinary",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().SourceDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFilesBinary",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().TargetDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
]
}
]
}
}
]
}
},
{
"name": "Log Upload Start",
"type": "Lookup",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[dbo].[LogUploadStart]",
"storedProcedureParameters": {
"FileCount": {
"type": "Int32",
"value": {
"value": "@activity('Get File List').output.count",
"type": "Expression"
}
},
"TriggerId": {
"type": "String",
"value": {
"value": "@pipeline().TriggerId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Log Upload Finish",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "UploadFiles",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[dbo].[LogUploadFinish]",
"storedProcedureParameters": {
"LogId": {
"value": {
"value": "@activity('Log Upload Start').output.firstRow.LogId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"variables": {
"DemoVariable": {
"type": "String"
}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-09-02T14:13:14Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]",
"[concat(variables('factoryId'), '/datasets/LaptopFilesBinary')]",
"[concat(variables('factoryId'), '/datasets/LakeFilesBinary')]",
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]",
"[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]",
"[concat(variables('factoryId'), '/datasets/LakeFileParquet')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/05 - Get File List Utility')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": {
"value": "@pipeline().parameters.GetFileQuery",
"type": "Expression"
},
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"parameters": {
"GetFileQuery": {
"type": "string"
}
},
"variables": {
"DemoVariable": {
"type": "String"
}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-09-02T14:13:14Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/06 - Example Global Param')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "06 - Get Any File List",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "05 - Get File List Utility",
"type": "PipelineReference"
},
"waitOnCompletion": false,
"parameters": {
"GetFileQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;"
}
}
},
{
"name": "Set Pipe Output",
"type": "SetVariable",
"dependsOn": [
{
"activity": "06 - Get Any File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "WorkerOutput",
"value": {
"value": "@activity('06 - Get Any File List').output.pipelineRunId",
"type": "Expression"
}
}
},
{
"name": "Set Global Param",
"type": "SetVariable",
"dependsOn": [
{
"activity": "06 - Get Any File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "GlobalParam",
"value": {
"value": "@concat(pipeline().globalParameters.Environment,'Test')",
"type": "Expression"
}
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"variables": {
"WorkerOutput": {
"type": "String"
},
"GlobalParam": {
"type": "String"
}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/pipelines/05 - Get File List Utility')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/1950 Activities')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "50",
"type": "ForEach",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')",
"type": "Expression"
},
"batchCount": 50,
"activities": [
{
"name": "Wait 1 Minute 1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 2",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 3",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 4",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 5",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 6",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 7",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 8",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 9",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 10",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 11",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 12",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 13",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 14",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 15",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 16",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 17",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 18",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 19",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 20",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 21",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 22",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 23",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 24",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 25",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 26",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 27",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 28",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 29",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 30",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 31",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 32",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 33",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 34",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 35",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 36",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 37",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 38",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 39",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": [],
"lastPublishTime": "2022-02-03T14:39:03Z"
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/BuildDimProduct')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Run UpdateProductDimension",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "UpdateProductDimension",
"type": "DataFlowReference",
"parameters": {},
"datasetParameters": {
"Product": {
"FileName": "Product.tsv"
},
"ProductSubcategory": {
"FileName": "ProductSubcategory.tsv"
},
"ProductCategory": {
"FileName": "ProductCategory.tsv"
},
"WriteToDataLake": {}
}
},
"staging": {},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Labs"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/dataflows/UpdateProductDimension')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/CopyAWEntities')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Copy each AW entity",
"type": "ForEach",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@variables('Entities')",
"type": "Expression"
},
"isSequential": false,
"activities": [
{
"name": "Execute CopyAWEntity",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "CopyAWEntity",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Entity": {
"value": "@item()",
"type": "Expression"
}
}
}
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"variables": {
"Entities": {
"type": "Array",
"defaultValue": [
"Product",
"ProductSubcategory",
"ProductCategory"
]
}
},
"folder": {
"name": "Labs"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/pipelines/CopyAWEntity')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/CopyAWEntity')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Copy AW entity",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
]
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/HTTP_BIN_AdventureWorks')]",
"[concat(variables('factoryId'), '/datasets/ADLS_BIN_AdventureWorks')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/CopyAWProduct')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Copy AW Products to data lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AWProduct",
"type": "DatasetReference",
"parameters": {}
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AWProduct",
"type": "DatasetReference",
"parameters": {}
}
]
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Labs"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/HTTP_BIN_AWProduct')]",
"[concat(variables('factoryId'), '/datasets/ADLS_BIN_AWProduct')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Custom Activity')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "HelloWorld",
"type": "Custom",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"command": "CustomActivity.exe",
"resourceLinkedService": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
},
"folderPath": "customactivities",
"referenceObjects": {
"linkedServices": [],
"datasets": []
}
},
"linkedServiceName": {
"referenceName": "BatchForTraining01",
"type": "LinkedServiceReference"
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": [],
"lastPublishTime": "2022-02-03T14:39:04Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]",
"[concat(variables('factoryId'), '/linkedServices/BatchForTraining01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Secret",
"type": "WebActivity",
"dependsOn": [
{
"activity": "Set Key URL",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"url": {
"value": "@variables('CompleteSecretURL')",
"type": "Expression"
},
"method": "GET",
"headers": {},
"authentication": {
"type": "MSI",
"resource": "https://vault.azure.net"
}
}
},
{
"name": "Set Key URL",
"type": "SetVariable",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"variableName": "CompleteSecretURL",
"value": {
"value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')",
"type": "Expression"
}
}
},
{
"name": "Nested Key Vault URL",
"type": "Lookup",
"dependsOn": [
{
"activity": "Get Secret",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "AnyDatabaseTableAnyKeyVault",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": "ConnectionString-trainingdb01",
"SchemaName": "sys",
"TableName": "objects",
"KeyVaultURL": {
"value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')",
"type": "Expression"
}
}
}
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"parameters": {
"SecretName": {
"type": "string",
"defaultValue": "DemoKeyGetWithWebActivity"
}
},
"variables": {
"CompleteSecretURL": {
"type": "String"
}
},
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/AnyDatabaseTableAnyKeyVault')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Lazy Replication')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Table List",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": {
"value": "@pipeline().parameters.TableLookupQuery",
"type": "Expression"
},
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "AnyDatabaseTable",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": {
"value": "@pipeline().parameters.SourceConnectionSecret",
"type": "Expression"
},
"SchemaName": "sys",
"TableName": "objects"
}
},
"firstRowOnly": false
}
},
{
"name": "Copy Tables",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Table List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Table List').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Copy Table",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "AzureSqlSink",
"preCopyScript": {
"value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}",
"type": "Expression"
},
"tableOption": "autoCreate",
"disableMetricsCollection": false
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "AnyDatabaseTable",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": {
"value": "@pipeline().parameters.SourceConnectionSecret",
"type": "Expression"
},
"SchemaName": {
"value": "@item().SchemaName",
"type": "Expression"
},
"TableName": {
"value": "@item().TableName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "AnyDatabaseTable",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": {
"value": "@pipeline().parameters.TargetConnectionSecret",
"type": "Expression"
},
"SchemaName": {
"value": "@item().SchemaName",
"type": "Expression"
},
"TableName": {
"value": "@item().TableName",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"parameters": {
"SourceConnectionSecret": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb01"
},
"TargetConnectionSecret": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb02"
},
"TableLookupQuery": {
"type": "string",
"defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'"
}
},
"folder": {
"name": "Demo Pipelines/Dynamic Linked Services"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:43:42Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/AnyDatabaseTable')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Order Summary with Mapping')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Mapping Order Aggregation",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "MappingOrderAggregation",
"type": "DataFlowReference",
"parameters": {},
"datasetParameters": {
"OrderHeader": {},
"OrderLineDetails": {},
"OrderSummary": {}
}
},
"staging": {},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Flows"
},
"annotations": [],
"lastPublishTime": "2020-09-02T14:13:15Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/dataflows/MappingOrderAggregation')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Scale Out Level 1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Bucket Ids",
"type": "Lookup",
"dependsOn": [
{
"activity": "Reset And Log Start",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Reset Bucket Process Map",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[dbo].[GetBuckets]",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Call Buckets",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Bucket Ids",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Bucket Ids').output.value",
"type": "Expression"
},
"isSequential": false,
"batchCount": 50,
"activities": [
{
"name": "Call Level 2",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "Scale Out Level 2",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"BucketId": {
"value": "@{item().BucketId}",
"type": "Expression"
}
}
}
}
]
}
},
{
"name": "Log End",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Call Buckets",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Reset And Log Start",
"dependencyConditions": [
"Completed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[dbo].[SetBucketLogEntry]",
"storedProcedureParameters": {
"LogId": {
"value": {
"value": "@activity('Reset And Log Start').output.firstRow.LogId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
},
{
"name": "Reset And Log Start",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[dbo].[SetBucketLogEntry]",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
}
}
},
{
"name": "Reset Bucket Process Map",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[[dbo].[SetBucketProcesses]"
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Scaling Out"
},
"annotations": [],
"lastPublishTime": "2020-08-24T13:41:19Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]",
"[concat(variables('factoryId'), '/pipelines/Scale Out Level 2')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get Bucket Contents",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]",
"storedProcedureParameters": {
"BucketId": {
"type": "Int32",
"value": {
"value": "@pipeline().parameters.BucketId",
"type": "Expression"
}
}
},
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference",
"parameters": {}
},
"firstRowOnly": false
}
},
{
"name": "Execute Processes",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Bucket Contents",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Bucket Contents').output.value",
"type": "Expression"
},
"batchCount": 50,
"activities": [
{
"name": "Run Process",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": {
"value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"parameters": {
"BucketId": {
"type": "int"
}
},
"folder": {
"name": "Demo Pipelines/Scaling Out"
},
"annotations": [],
"lastPublishTime": "2020-08-24T13:41:19Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/GetSetMetadata')]",
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/Wait 10')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait 15",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 15
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/ADLS_BIN_AWProduct')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Product.csv",
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/ADLS_BIN_AdventureWorks')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().Entity}.tsv",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/ADLS_TSV_AdventureWorks')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"FileName": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().FileName",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnection",
"type": "LinkedServiceReference",
"parameters": {
"DBConnectionSecret": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
"type": "LinkedServiceReference",
"parameters": {
"KeyVaultURL": {
"value": "@dataset().KeyVaultURL",
"type": "Expression"
},
"KeyVaultSecretName": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
},
"KeyVaultURL": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/HTTP_BIN_AWProduct')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWProduct",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWProduct')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/HTTP_BIN_AdventureWorks')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().Entity}.csv",
"type": "Expression"
}
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/HTTP_AWGitHub')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderDetailLines.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "SalesOrderDetailID",
"type": "INT32"
},
{
"name": "OrderQty",
"type": "INT32"
},
{
"name": "ProductID",
"type": "INT32"
},
{
"name": "UnitPrice",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "UnitPriceDiscount",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "LineTotal",
"type": "DECIMAL",
"precision": 38,
"scale": 6
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFileOrderHeaderParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderHeader.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "RevisionNumber",
"type": "INT32"
},
{
"name": "OrderDate",
"type": "INT96"
},
{
"name": "DueDate",
"type": "INT96"
},
{
"name": "ShipDate",
"type": "INT96"
},
{
"name": "Status",
"type": "INT32"
},
{
"name": "OnlineOrderFlag",
"type": "BOOLEAN"
},
{
"name": "SalesOrderNumber",
"type": "UTF8"
},
{
"name": "PurchaseOrderNumber",
"type": "UTF8"
},
{
"name": "AccountNumber",
"type": "UTF8"
},
{
"name": "CustomerID",
"type": "INT32"
},
{
"name": "ShipToAddressID",
"type": "INT32"
},
{
"name": "BillToAddressID",
"type": "INT32"
},
{
"name": "ShipMethod",
"type": "UTF8"
},
{
"name": "CreditCardApprovalCode",
"type": "UTF8"
},
{
"name": "SubTotal",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TaxAmt",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Freight",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TotalDue",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Comment",
"type": "UTF8"
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFileParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Person.csv",
"folderPath": "Landing",
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFiles')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "Person.csv",
"folderPath": "ForUpload/People"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"compressionCodec": "snappy"
},
"schema": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFolders')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LaptopsFiles')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/TableOrderSummary')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "SalesOrderNumber",
"type": "varchar"
},
{
"name": "RecordCount",
"type": "int",
"precision": 10
}
],
"typeProperties": {
"schema": "dbo",
"table": "OrderSummary"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/trainingdb01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/ADLS_saintegrationpipelines')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('ADLS_saintegrationpipelines_properties_typeProperties_url')]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"parameters": {
"DBConnectionSecret": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": {
"value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]",
"type": "Expression"
}
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"parameters": {
"KeyVaultURL": {
"type": "string"
},
"KeyVaultSecretName": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "GenericKeys",
"type": "LinkedServiceReference",
"parameters": {
"baseUrl": {
"value": "@linkedService().KeyVaultURL",
"type": "Expression"
}
}
},
"secretName": {
"value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]",
"type": "Expression"
}
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/GenericKeys')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AzureDataLakeStorage1')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('AzureDataLakeStorage1_properties_typeProperties_url')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/AzureSqlDatabase1')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": "[parameters('AzureSqlDatabase1_connectionString')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/BatchForTraining01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBatch",
"typeProperties": {
"batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]",
"poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]",
"accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]",
"linkedServiceName": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/GenericKeys')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"description": "https://TrainingKeys01.vault.azure.net",
"parameters": {
"baseUrl": {
"type": "String"
}
},
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/HTTP_AWGitHub')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "[parameters('HTTP_AWGitHub_properties_typeProperties_url')]",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/HTTP_AWProduct')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "[parameters('HTTP_AWProduct_properties_typeProperties_url')]",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFiles')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "FileServer",
"typeProperties": {
"host": "[parameters('LaptopFiles_properties_typeProperties_host')]",
"userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]",
"password": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ADFIRUser"
}
},
"connectVia": {
"referenceName": "PaulsLaptopIR",
"type": "IntegrationRuntimeReference"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]",
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/TrainingKeys01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]",
"credential": {
"referenceName": "TrainingCredential01",
"type": "CredentialReference"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/credentials/TrainingCredential01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/TrainingStore01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobStorage",
"typeProperties": {
"connectionString": "[parameters('TrainingStore01_connectionString')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/trainingdb01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/traininglak01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglak01_properties_typeProperties_url')]",
"accountKey": {
"type": "SecureString",
"value": "[parameters('traininglak01_accountKey')]"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/traininglake01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglake01_properties_typeProperties_url')]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/traininglake01noneKV')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglake01noneKV_properties_typeProperties_url')]",
"accountKey": {
"type": "SecureString",
"value": "[parameters('traininglake01noneKV_accountKey')]"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 240
}
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/PaulsLaptopIR')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "SelfHosted",
"typeProperties": {
"linkedInfo": {
"resourceId": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]",
"authorizationType": "Rbac"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/SSISIntegrationRuntime')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"nodeSize": "Standard_D2_v3",
"numberOfNodes": 1,
"maxParallelExecutionsPerNode": 2
},
"ssisProperties": {
"catalogInfo": {
"catalogServerEndpoint": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogServerEndpoint')]",
"catalogAdminUserName": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminUserName')]",
"catalogAdminPassword": {
"type": "SecureString",
"value": "[parameters('SSISIntegrationRuntime_properties_typeProperties_ssisProperties_catalogInfo_catalogAdminPassword')]"
},
"catalogPricingTier": "S1"
},
"edition": "Standard",
"licenseType": "LicenseIncluded"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 0,
"cleanup": true
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/MappingOrderAggregation')]",
"type": "Microsoft.DataFactory/factories/dataflows",
"apiVersion": "2018-06-01",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "LakeFileOrderHeaderParquet",
"type": "DatasetReference"
},
"name": "OrderHeader"
},
{
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
},
"name": "OrderLineDetails"
}
],
"sinks": [
{
"dataset": {
"referenceName": "TableOrderSummary",
"type": "DatasetReference"
},
"name": "OrderSummary"
}
],
"transformations": [
{
"name": "JoinHeaderToLineDetails"
},
{
"name": "OrderLineCount"
}
],
"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tmatchType:'exact',\n\tignoreSpaces: false,\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LakeFileOrderHeaderParquet')]",
"[concat(variables('factoryId'), '/datasets/LakeFileOrderDetailLinesParquet')]",
"[concat(variables('factoryId'), '/datasets/TableOrderSummary')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/UpdateProductDimension')]",
"type": "Microsoft.DataFactory/factories/dataflows",
"apiVersion": "2018-06-01",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/ADLS_TSV_AdventureWorks')]",
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/UpdateProductDimension_Sorted')]",
"type": "Microsoft.DataFactory/factories/dataflows",
"apiVersion": "2018-06-01",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
},
{
"name": "SortBySubcategory"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
"SortBySubcategory sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/ADLS_TSV_AdventureWorks')]",
"[concat(variables('factoryId'), '/linkedServices/ADLS_saintegrationpipelines')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]",
"type": "Microsoft.DataFactory/factories/dataflows",
"apiVersion": "2018-06-01",
"properties": {
"type": "WranglingDataFlow",
"typeProperties": {
"sources": [
{
"name": "LakeFileOrderDetailLinesParquet",
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
}
}
],
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LakeFileOrderDetailLinesParquet')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/TrainingCredential01')]",
"type": "Microsoft.DataFactory/factories/credentials",
"apiVersion": "2018-06-01",
"properties": {
"type": "ManagedIdentity",
"typeProperties": {
"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/default')]",
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks",
"apiVersion": "2018-06-01",
"properties": {},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]",
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
"apiVersion": "2018-06-01",
"properties": {
"privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]",
"groupId": "[parameters('AzureKeyVault1_properties_groupId')]",
"fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]"
},
"dependsOn": [
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
]
}
]
}
Reference in New Issue View Git Blame Copy Permalink