{ "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "factoryName": { "type": "string", "metadata": "Data Factory name", "defaultValue": "TrainingFactoryDev" }, "TrainingStore01_connectionString": { "type": "secureString", "metadata": "Secure string for 'connectionString' of 'TrainingStore01'" }, "traininglak01_accountKey": { "type": "secureString", "metadata": "Secure string for 'accountKey' of 'traininglak01'" }, "traininglake01noneKV_accountKey": { "type": "secureString", "metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'" }, "AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "@linkedService().DBConnectionSecret" }, "AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "@linkedService().KeyVaultSecretName" }, "BatchForTraining01_properties_typeProperties_batchUri": { "type": "string", "defaultValue": "https://batchfortraining01.uksouth.batch.azure.com" }, "BatchForTraining01_properties_typeProperties_poolName": { "type": "string", "defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654" }, "BatchForTraining01_properties_typeProperties_accountName": { "type": "string", "defaultValue": "batchfortraining01" }, "GenericKeys_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "@{linkedService().baseUrl}" }, "LaptopFiles_properties_typeProperties_host": { "type": "string", "defaultValue": "C:\\ADFRoot\\" }, "LaptopFiles_properties_typeProperties_userId": { "type": "string", "defaultValue": "ADFIRUser" }, "TrainingKeys01_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "https://TrainingKeys01.vault.azure.net/" }, "TrainingKeys01withUMI_properties_typeProperties_baseUrl": { "type": "string", "defaultValue": "https://TrainingKeys01.vault.azure.net/" }, "trainingdb01_properties_typeProperties_connectionString_secretName": { "type": "string", "defaultValue": "ConnectionString-trainingdb01" }, "traininglak01_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "traininglake01_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "traininglake01noneKV_properties_typeProperties_url": { "type": "string", "defaultValue": "https://traininglake01.dfs.core.windows.net" }, "PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": { "type": "string", "defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV" }, "AzureKeyVault1_properties_privateLinkResourceId": { "type": "string", "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" }, "AzureKeyVault1_properties_groupId": { "type": "string", "defaultValue": "vault" }, "AzureKeyVault1_properties_fqdns": { "type": "array", "defaultValue": [ "trainingkeys01.vaultcore.azure.net" ] } }, "variables": { "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" }, "resources": [ { "name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Upload Person", "description": "hfdhgfdhfd", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "DelimitedTextSource", "storeSettings": { "type": "FileServerReadSettings", "recursive": true }, "formatSettings": { "type": "DelimitedTextReadSettings" } }, "sink": { "type": "DelimitedTextSink", "storeSettings": { "type": "AzureBlobFSWriteSettings" }, "formatSettings": { "type": "DelimitedTextWriteSettings", "quoteAllText": true, "fileExtension": ".txt" } }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } }, "inputs": [ { "referenceName": "LaptopFilePersonCSV", "type": "DatasetReference", "parameters": {} } ], "outputs": [ { "referenceName": "LakeFilePersonCSV", "type": "DatasetReference", "parameters": {} } ] } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Data Uploads" }, "annotations": [], "lastPublishTime": "2020-09-29T13:50:28Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]", "[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]" ] }, { "name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Upload Any File", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "DelimitedTextSource", "storeSettings": { "type": "FileServerReadSettings", "recursive": true }, "formatSettings": { "type": "DelimitedTextReadSettings" } }, "sink": { "type": "DelimitedTextSink", "storeSettings": { "type": "AzureBlobFSWriteSettings" }, "formatSettings": { "type": "DelimitedTextWriteSettings", "quoteAllText": true, "fileExtension": ".txt" } }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } }, "inputs": [ { "referenceName": "LaptopsFiles", "type": "DatasetReference", "parameters": { "Directory": "ForUpload\\People", "File": "Person.csv" } } ], "outputs": [ { "referenceName": "LakeFiles", "type": "DatasetReference", "parameters": { "Directory": "Landing", "File": "Person.csv" } } ] } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Data Uploads" }, "annotations": [], "lastPublishTime": "2020-08-27T09:06:46Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", "[concat(variables('factoryId'), '/datasets/LakeFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get File List", "type": "GetMetadata", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataset": { "referenceName": "LaptopFolders", "type": "DatasetReference", "parameters": { "Directory": "ForUpload\\People" } }, "fieldList": [ "childItems" ], "storeSettings": { "type": "FileServerReadSettings", "recursive": true }, "formatSettings": { "type": "DelimitedTextReadSettings" } } }, { "name": "Upload Files", "type": "ForEach", "dependsOn": [ { "activity": "Get File List", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get File List').output.childItems", "type": "Expression" }, "activities": [ { "name": "Upload File", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "DelimitedTextSource", "storeSettings": { "type": "FileServerReadSettings", "recursive": true }, "formatSettings": { "type": "DelimitedTextReadSettings" } }, "sink": { "type": "DelimitedTextSink", "storeSettings": { "type": "AzureBlobFSWriteSettings" }, "formatSettings": { "type": "DelimitedTextWriteSettings", "quoteAllText": true, "fileExtension": ".txt" } }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } }, "inputs": [ { "referenceName": "LaptopsFiles", "type": "DatasetReference", "parameters": { "Directory": "ForUpload\\People", "File": { "value": "@item().name", "type": "Expression" } } } ], "outputs": [ { "referenceName": "LakeFiles", "type": "DatasetReference", "parameters": { "Directory": "Landing", "File": { "value": "@item().name", "type": "Expression" } } } ] } ] } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Data Uploads" }, "annotations": [], "lastPublishTime": "2020-08-27T09:06:45Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/LaptopFolders')]", "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", "[concat(variables('factoryId'), '/datasets/LakeFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get File List", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;", "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "UploadFiles", "type": "ForEach", "dependsOn": [ { "activity": "Get File List", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Log Upload Start", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get File List').output.value", "type": "Expression" }, "activities": [ { "name": "Copy By File Type", "type": "Switch", "dependsOn": [], "userProperties": [], "typeProperties": { "on": { "value": "@substring(item().FileName,add(indexof(item().FileName,'.'),1),sub(length(item().FileName),add(indexof(item().FileName,'.'),1)))", "type": "Expression" }, "cases": [ { "value": "csv", "activities": [ { "name": "CSV Copy", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "DelimitedTextSource", "additionalColumns": [ { "name": "FilePath", "value": "$$FILEPATH" }, { "name": "Environment", "value": { "value": "@pipeline().globalParameters.Environment", "type": "Expression" } }, { "name": "PipelineName", "value": { "value": "@pipeline().Pipeline", "type": "Expression" } }, { "name": "RunId", "value": { "value": "@pipeline().RunId", "type": "Expression" } }, { "name": "RunDate", "value": { "value": "@utcnow()", "type": "Expression" } } ], "storeSettings": { "type": "FileServerReadSettings", "recursive": false, "enablePartitionDiscovery": false }, "formatSettings": { "type": "DelimitedTextReadSettings" } }, "sink": { "type": "DelimitedTextSink", "storeSettings": { "type": "AzureBlobFSWriteSettings" }, "formatSettings": { "type": "DelimitedTextWriteSettings", "quoteAllText": true, "fileExtension": ".txt" } }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } }, "inputs": [ { "referenceName": "LaptopsFiles", "type": "DatasetReference", "parameters": { "Directory": { "value": "@item().SourceDirectory", "type": "Expression" }, "File": { "value": "@item().FileName", "type": "Expression" } } } ], "outputs": [ { "referenceName": "LakeFiles", "type": "DatasetReference", "parameters": { "Directory": { "value": "@item().TargetDirectory", "type": "Expression" }, "File": { "value": "@item().FileName", "type": "Expression" } } } ] } ] }, { "value": "parquet", "activities": [ { "name": "Parquet Copy", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "ParquetSource", "storeSettings": { "type": "FileServerReadSettings", "recursive": false, "enablePartitionDiscovery": false } }, "sink": { "type": "ParquetSink", "storeSettings": { "type": "AzureBlobFSWriteSettings" }, "formatSettings": { "type": "ParquetWriteSettings" } }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } }, "inputs": [ { "referenceName": "LaptopFilesParquet", "type": "DatasetReference", "parameters": { "Directory": { "value": "@item().SourceDirectory", "type": "Expression" }, "File": { "value": "@item().FileName", "type": "Expression" } } } ], "outputs": [ { "referenceName": "LakeFileParquet", "type": "DatasetReference", "parameters": { "Directory": { "value": "@item().TargetDirectory", "type": "Expression" }, "File": { "value": "@item().FileName", "type": "Expression" } } } ] } ] } ], "defaultActivities": [ { "name": "Binary Copy", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "BinarySource", "storeSettings": { "type": "FileServerReadSettings", "recursive": false }, "formatSettings": { "type": "BinaryReadSettings" } }, "sink": { "type": "BinarySink", "storeSettings": { "type": "AzureBlobFSWriteSettings" } }, "enableStaging": false }, "inputs": [ { "referenceName": "LaptopFilesBinary", "type": "DatasetReference", "parameters": { "Directory": { "value": "@item().SourceDirectory", "type": "Expression" }, "File": { "value": "@item().FileName", "type": "Expression" } } } ], "outputs": [ { "referenceName": "LakeFilesBinary", "type": "DatasetReference", "parameters": { "Directory": { "value": "@item().TargetDirectory", "type": "Expression" }, "File": { "value": "@item().FileName", "type": "Expression" } } } ] } ] } } ] } }, { "name": "Log Upload Start", "type": "Lookup", "dependsOn": [ { "activity": "Get File List", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[dbo].[LogUploadStart]", "storedProcedureParameters": { "FileCount": { "type": "Int32", "value": { "value": "@activity('Get File List').output.count", "type": "Expression" } }, "TriggerId": { "type": "String", "value": { "value": "@pipeline().TriggerId", "type": "Expression" } } }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Log Upload Finish", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "UploadFiles", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[LogUploadFinish]", "storedProcedureParameters": { "LogId": { "value": { "value": "@activity('Log Upload Start').output.firstRow.LogId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "variables": { "DemoVariable": { "type": "String" } }, "folder": { "name": "Demo Pipelines/Data Uploads" }, "annotations": [], "lastPublishTime": "2020-09-02T14:13:14Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]", "[concat(variables('factoryId'), '/datasets/LaptopFilesBinary')]", "[concat(variables('factoryId'), '/datasets/LakeFilesBinary')]", "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", "[concat(variables('factoryId'), '/datasets/LakeFiles')]", "[concat(variables('factoryId'), '/datasets/LaptopFilesParquet')]", "[concat(variables('factoryId'), '/datasets/LakeFileParquet')]" ] }, { "name": "[concat(parameters('factoryName'), '/05 - Get File List Utility')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get File List", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "@pipeline().parameters.GetFileQuery", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "parameters": { "GetFileQuery": { "type": "string" } }, "variables": { "DemoVariable": { "type": "String" } }, "folder": { "name": "Demo Pipelines/Data Uploads" }, "annotations": [], "lastPublishTime": "2020-09-02T14:13:14Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]" ] }, { "name": "[concat(parameters('factoryName'), '/06 - Example Global Param')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "06 - Get Any File List", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "05 - Get File List Utility", "type": "PipelineReference" }, "waitOnCompletion": false, "parameters": { "GetFileQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;" } } }, { "name": "Set Pipe Output", "type": "SetVariable", "dependsOn": [ { "activity": "06 - Get Any File List", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "WorkerOutput", "value": { "value": "@activity('06 - Get Any File List').output.pipelineRunId", "type": "Expression" } } }, { "name": "Set Global Param", "type": "SetVariable", "dependsOn": [ { "activity": "06 - Get Any File List", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "variableName": "GlobalParam", "value": { "value": "@concat(pipeline().globalParameters.Environment,'Test')", "type": "Expression" } } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "variables": { "WorkerOutput": { "type": "String" }, "GlobalParam": { "type": "String" } }, "folder": { "name": "Demo Pipelines/Data Uploads" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/pipelines/05 - Get File List Utility')]" ] }, { "name": "[concat(parameters('factoryName'), '/1950 Activities')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "50", "type": "ForEach", "dependsOn": [], "userProperties": [], "typeProperties": { "items": { "value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')", "type": "Expression" }, "batchCount": 50, "activities": [ { "name": "Wait 1 Minute 1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 2", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 3", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 4", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 5", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 6", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 7", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 8", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 9", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 10", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 11", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 12", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 13", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 14", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 15", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 16", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 17", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 18", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 19", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 20", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 21", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 22", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 23", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 24", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 25", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 26", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 27", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 28", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 29", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 30", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 31", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 32", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 33", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 34", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 35", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 36", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 37", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 38", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } }, { "name": "Wait 1 Minute 39", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 600 } } ] } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Misc" }, "annotations": [], "lastPublishTime": "2022-02-03T14:39:03Z" }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/Custom Activity')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "HelloWorld", "type": "Custom", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "command": "CustomActivity.exe", "resourceLinkedService": { "referenceName": "TrainingStore01", "type": "LinkedServiceReference" }, "folderPath": "customactivities", "referenceObjects": { "linkedServices": [], "datasets": [] } }, "linkedServiceName": { "referenceName": "BatchForTraining01", "type": "LinkedServiceReference" } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Misc" }, "annotations": [], "lastPublishTime": "2022-02-03T14:39:04Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]", "[concat(variables('factoryId'), '/linkedServices/BatchForTraining01')]" ] }, { "name": "[concat(parameters('factoryName'), '/Get Key Vault Value')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Secret", "type": "WebActivity", "dependsOn": [ { "activity": "Set Key URL", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "url": { "value": "@variables('CompleteSecretURL')", "type": "Expression" }, "method": "GET", "headers": {}, "authentication": { "type": "MSI", "resource": "https://vault.azure.net" } } }, { "name": "Set Key URL", "type": "SetVariable", "dependsOn": [], "userProperties": [], "typeProperties": { "variableName": "CompleteSecretURL", "value": { "value": "@concat(\n'https://',\npipeline().globalParameters.KeyVaultName,\n'.vault.azure.net/secrets/',\npipeline().parameters.SecretName,\n'?api-version=7.0')", "type": "Expression" } } }, { "name": "Nested Key Vault URL", "type": "Lookup", "dependsOn": [ { "activity": "Get Secret", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "AnyDatabaseTableAnyKeyVault", "type": "DatasetReference", "parameters": { "LinkedServiceConnectionSecret": "ConnectionString-trainingdb01", "SchemaName": "sys", "TableName": "objects", "KeyVaultURL": { "value": "@concat('https://',pipeline().globalParameters.KeyVaultName,'.vault.azure.net/')", "type": "Expression" } } } } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "parameters": { "SecretName": { "type": "string", "defaultValue": "DemoKeyGetWithWebActivity" } }, "variables": { "CompleteSecretURL": { "type": "String" } }, "folder": { "name": "Demo Pipelines/Misc" }, "annotations": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/AnyDatabaseTableAnyKeyVault')]" ] }, { "name": "[concat(parameters('factoryName'), '/Lazy Replication')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Table List", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderQuery": { "value": "@pipeline().parameters.TableLookupQuery", "type": "Expression" }, "queryTimeout": "02:00:00", "partitionOption": "None" }, "dataset": { "referenceName": "AnyDatabaseTable", "type": "DatasetReference", "parameters": { "LinkedServiceConnectionSecret": { "value": "@pipeline().parameters.SourceConnectionSecret", "type": "Expression" }, "SchemaName": "sys", "TableName": "objects" } }, "firstRowOnly": false } }, { "name": "Copy Tables", "type": "ForEach", "dependsOn": [ { "activity": "Get Table List", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Table List').output.value", "type": "Expression" }, "activities": [ { "name": "Copy Table", "type": "Copy", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "queryTimeout": "02:00:00", "partitionOption": "None" }, "sink": { "type": "AzureSqlSink", "preCopyScript": { "value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}", "type": "Expression" }, "tableOption": "autoCreate", "disableMetricsCollection": false }, "enableStaging": false, "translator": { "type": "TabularTranslator", "typeConversion": true, "typeConversionSettings": { "allowDataTruncation": true, "treatBooleanAsNumber": false } } }, "inputs": [ { "referenceName": "AnyDatabaseTable", "type": "DatasetReference", "parameters": { "LinkedServiceConnectionSecret": { "value": "@pipeline().parameters.SourceConnectionSecret", "type": "Expression" }, "SchemaName": { "value": "@item().SchemaName", "type": "Expression" }, "TableName": { "value": "@item().TableName", "type": "Expression" } } } ], "outputs": [ { "referenceName": "AnyDatabaseTable", "type": "DatasetReference", "parameters": { "LinkedServiceConnectionSecret": { "value": "@pipeline().parameters.TargetConnectionSecret", "type": "Expression" }, "SchemaName": { "value": "@item().SchemaName", "type": "Expression" }, "TableName": { "value": "@item().TableName", "type": "Expression" } } } ] } ] } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "parameters": { "SourceConnectionSecret": { "type": "string", "defaultValue": "ConnectionString-trainingdb01" }, "TargetConnectionSecret": { "type": "string", "defaultValue": "ConnectionString-trainingdb02" }, "TableLookupQuery": { "type": "string", "defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'" } }, "folder": { "name": "Demo Pipelines/Dynamic Linked Services" }, "annotations": [], "lastPublishTime": "2020-08-27T09:43:42Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/AnyDatabaseTable')]" ] }, { "name": "[concat(parameters('factoryName'), '/Order Summary with Mapping')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Mapping Order Aggregation", "type": "ExecuteDataFlow", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "dataflow": { "referenceName": "MappingOrderAggregation", "type": "DataFlowReference", "parameters": {}, "datasetParameters": { "OrderHeader": {}, "OrderLineDetails": {}, "OrderSummary": {} } }, "staging": {}, "compute": { "coreCount": 8, "computeType": "General" }, "traceLevel": "Fine" } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Data Flows" }, "annotations": [], "lastPublishTime": "2020-09-02T14:13:15Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/dataflows/MappingOrderAggregation')]" ] }, { "name": "[concat(parameters('factoryName'), '/Scale Out Level 1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Bucket Ids", "type": "Lookup", "dependsOn": [ { "activity": "Reset And Log Start", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Reset Bucket Process Map", "dependencyConditions": [ "Succeeded" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[dbo].[GetBuckets]", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Call Buckets", "type": "ForEach", "dependsOn": [ { "activity": "Get Bucket Ids", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Bucket Ids').output.value", "type": "Expression" }, "isSequential": false, "batchCount": 50, "activities": [ { "name": "Call Level 2", "type": "ExecutePipeline", "dependsOn": [], "userProperties": [], "typeProperties": { "pipeline": { "referenceName": "Scale Out Level 2", "type": "PipelineReference" }, "waitOnCompletion": true, "parameters": { "BucketId": { "value": "@{item().BucketId}", "type": "Expression" } } } } ] } }, { "name": "Log End", "type": "SqlServerStoredProcedure", "dependsOn": [ { "activity": "Call Buckets", "dependencyConditions": [ "Succeeded" ] }, { "activity": "Reset And Log Start", "dependencyConditions": [ "Completed" ] } ], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetBucketLogEntry]", "storedProcedureParameters": { "LogId": { "value": { "value": "@activity('Reset And Log Start').output.firstRow.LogId", "type": "Expression" }, "type": "Int32" } } }, "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" } }, { "name": "Reset And Log Start", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[dbo].[SetBucketLogEntry]", "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} } } }, { "name": "Reset Bucket Process Map", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": "[[dbo].[SetBucketProcesses]" }, "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Scaling Out" }, "annotations": [], "lastPublishTime": "2020-08-24T13:41:19Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]", "[concat(variables('factoryId'), '/pipelines/Scale Out Level 2')]" ] }, { "name": "[concat(parameters('factoryName'), '/Scale Out Level 2')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Get Bucket Contents", "type": "Lookup", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "source": { "type": "AzureSqlSource", "sqlReaderStoredProcedureName": "[[dbo].[GetBucketContents]", "storedProcedureParameters": { "BucketId": { "type": "Int32", "value": { "value": "@pipeline().parameters.BucketId", "type": "Expression" } } }, "partitionOption": "None" }, "dataset": { "referenceName": "GetSetMetadata", "type": "DatasetReference", "parameters": {} }, "firstRowOnly": false } }, { "name": "Execute Processes", "type": "ForEach", "dependsOn": [ { "activity": "Get Bucket Contents", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "typeProperties": { "items": { "value": "@activity('Get Bucket Contents').output.value", "type": "Expression" }, "batchCount": 50, "activities": [ { "name": "Run Process", "type": "SqlServerStoredProcedure", "dependsOn": [], "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureOutput": false, "secureInput": false }, "userProperties": [], "typeProperties": { "storedProcedureName": { "value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')", "type": "Expression" } }, "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" } } ] } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "parameters": { "BucketId": { "type": "int" } }, "folder": { "name": "Demo Pipelines/Scaling Out" }, "annotations": [], "lastPublishTime": "2020-08-24T13:41:19Z" }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" ] }, { "name": "[concat(parameters('factoryName'), '/Wait 10')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait 15", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 15 } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "folder": { "name": "Demo Pipelines/Misc" }, "annotations": [] }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "AnyDatabaseConnection", "type": "LinkedServiceReference", "parameters": { "DBConnectionSecret": { "value": "@dataset().LinkedServiceConnectionSecret", "type": "Expression" } } }, "parameters": { "LinkedServiceConnectionSecret": { "type": "string" }, "SchemaName": { "type": "string" }, "TableName": { "type": "string" } }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": { "schema": { "value": "@dataset().SchemaName", "type": "Expression" }, "table": { "value": "@dataset().TableName", "type": "Expression" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnection')]" ] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "AnyDatabaseConnectionFromAnyKeyVault", "type": "LinkedServiceReference", "parameters": { "KeyVaultURL": { "value": "@dataset().KeyVaultURL", "type": "Expression" }, "KeyVaultSecretName": { "value": "@dataset().LinkedServiceConnectionSecret", "type": "Expression" } } }, "parameters": { "LinkedServiceConnectionSecret": { "type": "string" }, "SchemaName": { "type": "string" }, "TableName": { "type": "string" }, "KeyVaultURL": { "type": "string" } }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": { "schema": { "value": "@dataset().SchemaName", "type": "Expression" }, "table": { "value": "@dataset().TableName", "type": "Expression" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/AnyDatabaseConnectionFromAnyKeyVault')]" ] }, { "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [], "typeProperties": {} }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01noneKV", "type": "LinkedServiceReference" }, "folder": { "name": "Lake" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "OrderDetailLines.parquet", "folderPath": "Raw", "fileSystem": "datawarehouse" }, "compressionCodec": "snappy" }, "schema": [ { "name": "SalesOrderID", "type": "INT32" }, { "name": "SalesOrderDetailID", "type": "INT32" }, { "name": "OrderQty", "type": "INT32" }, { "name": "ProductID", "type": "INT32" }, { "name": "UnitPrice", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "UnitPriceDiscount", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "LineTotal", "type": "DECIMAL", "precision": 38, "scale": 6 }, { "name": "rowguid", "type": "UTF8" }, { "name": "ModifiedDate", "type": "INT96" } ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFileOrderHeaderParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01noneKV", "type": "LinkedServiceReference" }, "folder": { "name": "Lake" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "OrderHeader.parquet", "folderPath": "Raw", "fileSystem": "datawarehouse" }, "compressionCodec": "snappy" }, "schema": [ { "name": "SalesOrderID", "type": "INT32" }, { "name": "RevisionNumber", "type": "INT32" }, { "name": "OrderDate", "type": "INT96" }, { "name": "DueDate", "type": "INT96" }, { "name": "ShipDate", "type": "INT96" }, { "name": "Status", "type": "INT32" }, { "name": "OnlineOrderFlag", "type": "BOOLEAN" }, { "name": "SalesOrderNumber", "type": "UTF8" }, { "name": "PurchaseOrderNumber", "type": "UTF8" }, { "name": "AccountNumber", "type": "UTF8" }, { "name": "CustomerID", "type": "INT32" }, { "name": "ShipToAddressID", "type": "INT32" }, { "name": "BillToAddressID", "type": "INT32" }, { "name": "ShipMethod", "type": "UTF8" }, { "name": "CreditCardApprovalCode", "type": "UTF8" }, { "name": "SubTotal", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "TaxAmt", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "Freight", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "TotalDue", "type": "DECIMAL", "precision": 19, "scale": 4 }, { "name": "Comment", "type": "UTF8" }, { "name": "rowguid", "type": "UTF8" }, { "name": "ModifiedDate", "type": "INT96" } ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01noneKV')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFileParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Lake" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" }, "fileSystem": "datawarehouse" }, "compressionCodec": "snappy" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "folder": { "name": "Lake" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": "Person.csv", "folderPath": "Landing", "fileSystem": "datawarehouse" }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [ { "type": "String" }, { "type": "String" }, { "type": "String" } ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFiles')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Lake" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" }, "fileSystem": "datawarehouse" }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "traininglake01", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Lake" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "AzureBlobFSLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" }, "fileSystem": "datawarehouse" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/traininglake01')]" ] }, { "name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "LaptopFiles", "type": "LinkedServiceReference" }, "folder": { "name": "Laptop" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "FileServerLocation", "fileName": "Person.csv", "folderPath": "ForUpload/People" }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [ { "type": "String" }, { "type": "String" }, { "type": "String" } ] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "LaptopFiles", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Laptop" }, "annotations": [], "type": "Binary", "typeProperties": { "location": { "type": "FileServerLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" } } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "LaptopFiles", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Laptop" }, "annotations": [], "type": "Parquet", "typeProperties": { "location": { "type": "FileServerLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" } }, "compressionCodec": "snappy" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/LaptopFolders')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "LaptopFiles", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" } }, "folder": { "name": "Laptop" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "FileServerLocation", "folderPath": { "value": "@dataset().Directory", "type": "Expression" } }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/LaptopsFiles')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "LaptopFiles", "type": "LinkedServiceReference" }, "parameters": { "Directory": { "type": "string" }, "File": { "type": "string" } }, "folder": { "name": "Laptop" }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "FileServerLocation", "fileName": { "value": "@dataset().File", "type": "Expression" }, "folderPath": { "value": "@dataset().Directory", "type": "Expression" } }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/LaptopFiles')]" ] }, { "name": "[concat(parameters('factoryName'), '/TableOrderSummary')]", "type": "Microsoft.DataFactory/factories/datasets", "apiVersion": "2018-06-01", "properties": { "linkedServiceName": { "referenceName": "trainingdb01", "type": "LinkedServiceReference" }, "folder": { "name": "SQLDB" }, "annotations": [], "type": "AzureSqlTable", "schema": [ { "name": "SalesOrderNumber", "type": "varchar" }, { "name": "RecordCount", "type": "int", "precision": 10 } ], "typeProperties": { "schema": "dbo", "table": "OrderSummary" } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/trainingdb01')]" ] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnection')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "parameters": { "DBConnectionSecret": { "type": "string" } }, "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": { "value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]", "type": "Expression" } } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" ] }, { "name": "[concat(parameters('factoryName'), '/AnyDatabaseConnectionFromAnyKeyVault')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "parameters": { "KeyVaultURL": { "type": "string" }, "KeyVaultSecretName": { "type": "string" } }, "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "GenericKeys", "type": "LinkedServiceReference", "parameters": { "baseUrl": { "value": "@linkedService().KeyVaultURL", "type": "Expression" } } }, "secretName": { "value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]", "type": "Expression" } } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/GenericKeys')]" ] }, { "name": "[concat(parameters('factoryName'), '/BatchForTraining01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBatch", "typeProperties": { "batchUri": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]", "poolName": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]", "accountName": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]", "linkedServiceName": { "referenceName": "TrainingStore01", "type": "LinkedServiceReference" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/TrainingStore01')]" ] }, { "name": "[concat(parameters('factoryName'), '/GenericKeys')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "description": "https://TrainingKeys01.vault.azure.net", "parameters": { "baseUrl": { "type": "String" } }, "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/LaptopFiles')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "FileServer", "typeProperties": { "host": "[parameters('LaptopFiles_properties_typeProperties_host')]", "userId": "[parameters('LaptopFiles_properties_typeProperties_userId')]", "password": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "ADFIRUser" } }, "connectVia": { "referenceName": "PaulsLaptopIR", "type": "IntegrationRuntimeReference" } }, "dependsOn": [ "[concat(variables('factoryId'), '/integrationRuntimes/PaulsLaptopIR')]", "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" ] }, { "name": "[concat(parameters('factoryName'), '/TrainingKeys01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/TrainingKeys01withUMI')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureKeyVault", "typeProperties": { "baseUrl": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]", "credential": { "referenceName": "TrainingCredential01", "type": "CredentialReference" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/credentials/TrainingCredential01')]" ] }, { "name": "[concat(parameters('factoryName'), '/TrainingStore01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobStorage", "typeProperties": { "connectionString": "[parameters('TrainingStore01_connectionString')]" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/trainingdb01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureSqlDatabase", "typeProperties": { "connectionString": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" ] }, { "name": "[concat(parameters('factoryName'), '/traininglak01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('traininglak01_properties_typeProperties_url')]", "accountKey": { "type": "SecureString", "value": "[parameters('traininglak01_accountKey')]" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/traininglake01')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('traininglake01_properties_typeProperties_url')]", "accountKey": { "type": "AzureKeyVaultSecret", "store": { "referenceName": "TrainingKeys01", "type": "LinkedServiceReference" }, "secretName": "traininglake01" } } }, "dependsOn": [ "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" ] }, { "name": "[concat(parameters('factoryName'), '/traininglake01noneKV')]", "type": "Microsoft.DataFactory/factories/linkedServices", "apiVersion": "2018-06-01", "properties": { "annotations": [], "type": "AzureBlobFS", "typeProperties": { "url": "[parameters('traininglake01noneKV_properties_typeProperties_url')]", "accountKey": { "type": "SecureString", "value": "[parameters('traininglake01noneKV_accountKey')]" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/ForDataFlowDemos')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", "apiVersion": "2018-06-01", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "UK South", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 240 } } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/PaulsLaptopIR')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", "apiVersion": "2018-06-01", "properties": { "type": "SelfHosted", "typeProperties": { "linkedInfo": { "resourceId": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]", "authorizationType": "Rbac" } } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", "type": "Microsoft.DataFactory/factories/integrationRuntimes", "apiVersion": "2018-06-01", "properties": { "type": "Managed", "typeProperties": { "computeProperties": { "location": "UK South", "dataFlowProperties": { "computeType": "General", "coreCount": 8, "timeToLive": 0, "cleanup": true } } }, "managedVirtualNetwork": { "type": "ManagedVirtualNetworkReference", "referenceName": "default" } }, "dependsOn": [ "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" ] }, { "name": "[concat(parameters('factoryName'), '/MappingOrderAggregation')]", "type": "Microsoft.DataFactory/factories/dataflows", "apiVersion": "2018-06-01", "properties": { "type": "MappingDataFlow", "typeProperties": { "sources": [ { "dataset": { "referenceName": "LakeFileOrderHeaderParquet", "type": "DatasetReference" }, "name": "OrderHeader" }, { "dataset": { "referenceName": "LakeFileOrderDetailLinesParquet", "type": "DatasetReference" }, "name": "OrderLineDetails" } ], "sinks": [ { "dataset": { "referenceName": "TableOrderSummary", "type": "DatasetReference" }, "name": "OrderSummary" } ], "transformations": [ { "name": "JoinHeaderToLineDetails" }, { "name": "OrderLineCount" } ], "script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary" } }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/LakeFileOrderHeaderParquet')]", "[concat(variables('factoryId'), '/datasets/LakeFileOrderDetailLinesParquet')]", "[concat(variables('factoryId'), '/datasets/TableOrderSummary')]" ] }, { "name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]", "type": "Microsoft.DataFactory/factories/dataflows", "apiVersion": "2018-06-01", "properties": { "type": "WranglingDataFlow", "typeProperties": { "sources": [ { "name": "LakeFileOrderDetailLinesParquet", "script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet", "dataset": { "referenceName": "LakeFileOrderDetailLinesParquet", "type": "DatasetReference" } } ], "script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n" } }, "dependsOn": [ "[concat(variables('factoryId'), '/datasets/LakeFileOrderDetailLinesParquet')]" ] }, { "name": "[concat(parameters('factoryName'), '/TrainingCredential01')]", "type": "Microsoft.DataFactory/factories/credentials", "apiVersion": "2018-06-01", "properties": { "type": "ManagedIdentity", "typeProperties": { "resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI" } }, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/default')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks", "apiVersion": "2018-06-01", "properties": {}, "dependsOn": [] }, { "name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]", "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints", "apiVersion": "2018-06-01", "properties": { "privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]", "groupId": "[parameters('AzureKeyVault1_properties_groupId')]", "fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]" }, "dependsOn": [ "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" ] }, { "name": "[concat(parameters('factoryName'), '/pipeline1')]", "type": "Microsoft.DataFactory/factories/pipelines", "apiVersion": "2018-06-01", "properties": { "activities": [ { "name": "Wait1", "type": "Wait", "dependsOn": [], "userProperties": [], "typeProperties": { "waitTimeInSeconds": 1 } } ], "policy": { "elapsedTimeMetric": {}, "cancelAfter": {} }, "annotations": [] }, "dependsOn": [] } ] }