ARM template and parameters deployed on 2-26-2022 10:40:26, based on the collaboration branch's commit ID: 8d07f3c5d30e1d16f8bdfeccca7bcb39d5eb76a0

This commit is contained in:
Paul Andrew 2022-02-26 10:40:27 +00:00
parent c43f6e5932
commit d1c841b473
8 changed files with 6896 additions and 0 deletions

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,74 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"value": "TrainingFactoryDev"
},
"TrainingStore01_connectionString": {
"value": ""
},
"traininglak01_accountKey": {
"value": ""
},
"traininglake01noneKV_accountKey": {
"value": ""
},
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
"value": "@linkedService().DBConnectionSecret"
},
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
"value": "@linkedService().KeyVaultSecretName"
},
"BatchForTraining01_properties_typeProperties_batchUri": {
"value": "https://batchfortraining01.uksouth.batch.azure.com"
},
"BatchForTraining01_properties_typeProperties_poolName": {
"value": "09a2ec47-0cc8-4492-935c-04852e3ec654"
},
"BatchForTraining01_properties_typeProperties_accountName": {
"value": "batchfortraining01"
},
"GenericKeys_properties_typeProperties_baseUrl": {
"value": "@{linkedService().baseUrl}"
},
"LaptopFiles_properties_typeProperties_host": {
"value": "C:\\ADFRoot\\"
},
"LaptopFiles_properties_typeProperties_userId": {
"value": "ADFIRUser"
},
"TrainingKeys01_properties_typeProperties_baseUrl": {
"value": "https://TrainingKeys01.vault.azure.net/"
},
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
"value": "https://TrainingKeys01.vault.azure.net/"
},
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"value": "ConnectionString-trainingdb01"
},
"traininglak01_properties_typeProperties_url": {
"value": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01_properties_typeProperties_url": {
"value": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01noneKV_properties_typeProperties_url": {
"value": "https://traininglake01.dfs.core.windows.net"
},
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
"value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
},
"AzureKeyVault1_properties_groupId": {
"value": "vault"
},
"AzureKeyVault1_properties_fqdns": {
"value": [
"trainingkeys01.vaultcore.azure.net"
]
}
}
}

View File

@ -0,0 +1,10 @@
{
"Environment": {
"type": "string",
"value": "Dev"
},
"KeyVaultName": {
"type": "string",
"value": "trainingkeys01"
}
}

View File

@ -0,0 +1,80 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"value": "TrainingFactoryDev"
},
"TrainingStore01_connectionString": {
"value": ""
},
"traininglak01_accountKey": {
"value": ""
},
"traininglake01noneKV_accountKey": {
"value": ""
},
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
"value": "@linkedService().DBConnectionSecret"
},
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
"value": "@linkedService().KeyVaultSecretName"
},
"BatchForTraining01_properties_typeProperties_batchUri": {
"value": "https://batchfortraining01.uksouth.batch.azure.com"
},
"BatchForTraining01_properties_typeProperties_poolName": {
"value": "09a2ec47-0cc8-4492-935c-04852e3ec654"
},
"BatchForTraining01_properties_typeProperties_accountName": {
"value": "batchfortraining01"
},
"GenericKeys_properties_typeProperties_baseUrl": {
"value": "@{linkedService().baseUrl}"
},
"LaptopFiles_properties_typeProperties_host": {
"value": "C:\\ADFRoot\\"
},
"LaptopFiles_properties_typeProperties_userId": {
"value": "ADFIRUser"
},
"TrainingKeys01_properties_typeProperties_baseUrl": {
"value": "https://TrainingKeys01.vault.azure.net/"
},
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
"value": "https://TrainingKeys01.vault.azure.net/"
},
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"value": "ConnectionString-trainingdb01"
},
"traininglak01_properties_typeProperties_url": {
"value": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01_properties_typeProperties_url": {
"value": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01noneKV_properties_typeProperties_url": {
"value": "https://traininglake01.dfs.core.windows.net"
},
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
"value": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"value": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
},
"AzureKeyVault1_properties_groupId": {
"value": "vault"
},
"AzureKeyVault1_properties_fqdns": {
"value": [
"trainingkeys01.vaultcore.azure.net"
]
},
"containerUri": {
"value": ""
},
"containerSasToken": {
"value": ""
}
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,954 @@
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name",
"defaultValue": "TrainingFactoryDev"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"type": "string",
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
},
"AzureKeyVault1_properties_groupId": {
"type": "string",
"defaultValue": "vault"
},
"AzureKeyVault1_properties_fqdns": {
"type": "array",
"defaultValue": [
"trainingkeys01.vaultcore.azure.net"
]
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 0,
"cleanup": true
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]",
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
"apiVersion": "2018-06-01",
"properties": {
"privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]",
"groupId": "[parameters('AzureKeyVault1_properties_groupId')]",
"fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]"
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/Custom Activity')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "HelloWorld",
"type": "Custom",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"command": "CustomActivity.exe",
"resourceLinkedService": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
},
"folderPath": "customactivities",
"referenceObjects": {
"linkedServices": [],
"datasets": []
}
},
"linkedServiceName": {
"referenceName": "BatchForTraining01",
"type": "LinkedServiceReference"
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": [],
"lastPublishTime": "2022-02-03T14:39:04Z"
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnection",
"type": "LinkedServiceReference",
"parameters": {
"DBConnectionSecret": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
"type": "LinkedServiceReference",
"parameters": {
"KeyVaultURL": {
"value": "@dataset().KeyVaultURL",
"type": "Expression"
},
"KeyVaultSecretName": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
},
"KeyVaultURL": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LakeFileParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Person.csv",
"folderPath": "Landing",
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LakeFiles')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "Person.csv",
"folderPath": "ForUpload/People"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFolders')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LaptopsFiles')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/TableOrderSummary')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "SalesOrderNumber",
"type": "varchar"
},
{
"name": "RecordCount",
"type": "int",
"precision": 10
}
],
"typeProperties": {
"schema": "dbo",
"table": "OrderSummary"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]",
"type": "Microsoft.DataFactory/factories/dataflows",
"apiVersion": "2018-06-01",
"properties": {
"type": "WranglingDataFlow",
"typeProperties": {
"sources": [
{
"name": "LakeFileOrderDetailLinesParquet",
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
}
}
],
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"compressionCodec": "snappy"
},
"schema": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Upload Person",
"description": "hfdhgfdhfd",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopFilePersonCSV",
"type": "DatasetReference",
"parameters": {}
}
],
"outputs": [
{
"referenceName": "LakeFilePersonCSV",
"type": "DatasetReference",
"parameters": {}
}
]
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-09-29T13:50:28Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]",
"[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Upload Any File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": "Person.csv"
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": "Person.csv"
}
}
]
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:46Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopFolders')]",
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
}
]
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,231 @@
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory name",
"defaultValue": "TrainingFactoryDev"
},
"TrainingStore01_connectionString": {
"type": "secureString",
"metadata": "Secure string for 'connectionString' of 'TrainingStore01'"
},
"traininglak01_accountKey": {
"type": "secureString",
"metadata": "Secure string for 'accountKey' of 'traininglak01'"
},
"traininglake01noneKV_accountKey": {
"type": "secureString",
"metadata": "Secure string for 'accountKey' of 'traininglake01noneKV'"
},
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "@linkedService().DBConnectionSecret"
},
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "@linkedService().KeyVaultSecretName"
},
"BatchForTraining01_properties_typeProperties_batchUri": {
"type": "string",
"defaultValue": "https://batchfortraining01.uksouth.batch.azure.com"
},
"BatchForTraining01_properties_typeProperties_poolName": {
"type": "string",
"defaultValue": "09a2ec47-0cc8-4492-935c-04852e3ec654"
},
"BatchForTraining01_properties_typeProperties_accountName": {
"type": "string",
"defaultValue": "batchfortraining01"
},
"GenericKeys_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": "@{linkedService().baseUrl}"
},
"LaptopFiles_properties_typeProperties_host": {
"type": "string",
"defaultValue": "C:\\ADFRoot\\"
},
"LaptopFiles_properties_typeProperties_userId": {
"type": "string",
"defaultValue": "ADFIRUser"
},
"TrainingKeys01_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
},
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
"type": "string",
"defaultValue": "https://TrainingKeys01.vault.azure.net/"
},
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb01"
},
"traininglak01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"traininglake01noneKV_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
"type": "string",
"defaultValue": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"type": "string",
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
},
"AzureKeyVault1_properties_groupId": {
"type": "string",
"defaultValue": "vault"
},
"AzureKeyVault1_properties_fqdns": {
"type": "array",
"defaultValue": [
"trainingkeys01.vaultcore.azure.net"
]
},
"containerUri": {
"type": "string",
"metadata": "URI to storage account containing linked ARM templates"
},
"containerSasToken": {
"type": "string",
"metadata": "Sas token to storage account containing linked ARM templates"
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "TrainingFactoryDev_ArmTemplate_0",
"type": "Microsoft.Resources/deployments",
"apiVersion": "2018-02-01",
"properties": {
"mode": "incremental",
"templateLink": {
"uri": "[concat(parameters('containerUri'), '/ArmTemplate_0.json', parameters('containerSasToken'))]",
"contentVersion": "1.0.0.0"
},
"parameters": {
"factoryName": {
"value": "[parameters('factoryName')]"
},
"TrainingStore01_connectionString": {
"value": "[parameters('TrainingStore01_connectionString')]"
},
"traininglak01_accountKey": {
"value": "[parameters('traininglak01_accountKey')]"
},
"traininglake01noneKV_accountKey": {
"value": "[parameters('traininglake01noneKV_accountKey')]"
},
"GenericKeys_properties_typeProperties_baseUrl": {
"value": "[parameters('GenericKeys_properties_typeProperties_baseUrl')]"
},
"TrainingKeys01_properties_typeProperties_baseUrl": {
"value": "[parameters('TrainingKeys01_properties_typeProperties_baseUrl')]"
},
"traininglak01_properties_typeProperties_url": {
"value": "[parameters('traininglak01_properties_typeProperties_url')]"
},
"traininglake01noneKV_properties_typeProperties_url": {
"value": "[parameters('traininglake01noneKV_properties_typeProperties_url')]"
},
"PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId": {
"value": "[parameters('PaulsLaptopIR_properties_typeProperties_linkedInfo_resourceId')]"
},
"AnyDatabaseConnection_properties_typeProperties_connectionString_secretName": {
"value": "[parameters('AnyDatabaseConnection_properties_typeProperties_connectionString_secretName')]"
},
"AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName": {
"value": "[parameters('AnyDatabaseConnectionFromAnyKeyVault_properties_typeProperties_connectionString_secretName')]"
},
"BatchForTraining01_properties_typeProperties_batchUri": {
"value": "[parameters('BatchForTraining01_properties_typeProperties_batchUri')]"
},
"BatchForTraining01_properties_typeProperties_poolName": {
"value": "[parameters('BatchForTraining01_properties_typeProperties_poolName')]"
},
"BatchForTraining01_properties_typeProperties_accountName": {
"value": "[parameters('BatchForTraining01_properties_typeProperties_accountName')]"
},
"LaptopFiles_properties_typeProperties_host": {
"value": "[parameters('LaptopFiles_properties_typeProperties_host')]"
},
"LaptopFiles_properties_typeProperties_userId": {
"value": "[parameters('LaptopFiles_properties_typeProperties_userId')]"
},
"TrainingKeys01withUMI_properties_typeProperties_baseUrl": {
"value": "[parameters('TrainingKeys01withUMI_properties_typeProperties_baseUrl')]"
},
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
},
"traininglake01_properties_typeProperties_url": {
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
}
}
},
"dependsOn": []
},
{
"name": "TrainingFactoryDev_ArmTemplate_1",
"type": "Microsoft.Resources/deployments",
"apiVersion": "2018-02-01",
"properties": {
"mode": "incremental",
"templateLink": {
"uri": "[concat(parameters('containerUri'), '/ArmTemplate_1.json', parameters('containerSasToken'))]",
"contentVersion": "1.0.0.0"
},
"parameters": {
"factoryName": {
"value": "[parameters('factoryName')]"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]"
},
"AzureKeyVault1_properties_groupId": {
"value": "[parameters('AzureKeyVault1_properties_groupId')]"
},
"AzureKeyVault1_properties_fqdns": {
"value": "[parameters('AzureKeyVault1_properties_fqdns')]"
}
}
},
"dependsOn": [
"[resourceId('Microsoft.Resources/deployments', 'TrainingFactoryDev_ArmTemplate_0')]"
]
},
{
"name": "TrainingFactoryDev_ArmTemplate_2",
"type": "Microsoft.Resources/deployments",
"apiVersion": "2018-02-01",
"properties": {
"mode": "incremental",
"templateLink": {
"uri": "[concat(parameters('containerUri'), '/ArmTemplate_2.json', parameters('containerSasToken'))]",
"contentVersion": "1.0.0.0"
},
"parameters": {
"factoryName": {
"value": "[parameters('factoryName')]"
}
}
},
"dependsOn": [
"[resourceId('Microsoft.Resources/deployments', 'TrainingFactoryDev_ArmTemplate_1')]"
]
}
]
}