dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ARM template and parameters deployed on 3-9-2022 15:59:37, based on the collaboration branch's commit ID: 09c848bc7d65f8fe7a0f7efad419136c1941ef5d

Browse Source
This commit is contained in:
Paul Andrew 2022-03-09 15:59:39 +00:00
parent 104531fb1f
commit 2a23b67be2
5 changed files with 231 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
TrainingFactoryDev/ARMTemplateForFactory.json
View File

@ -3259,6 +3259,30 @@
"dependsOn": [
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/pipeline1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"annotations": []
},
"dependsOn": []
}
]
}

51
TrainingFactoryDev/linkedTemplates/ArmTemplate_0.json
View File

@ -74,10 +74,6 @@
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb01"
},
"traininglake01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
}
},
"variables": {
@ -628,6 +624,30 @@
"properties": {},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/pipeline1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"annotations": []
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]",
"type": "Microsoft.DataFactory/factories/datasets",
@ -985,29 +1005,6 @@
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/traininglake01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglake01_properties_typeProperties_url')]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
}
]
}

188
TrainingFactoryDev/linkedTemplates/ArmTemplate_1.json
View File

@ -7,6 +7,10 @@
"metadata": "Data Factory name",
"defaultValue": "TrainingFactoryDev"
},
"traininglake01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"type": "string",
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
@ -26,6 +30,27 @@
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/traininglake01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglake01_properties_typeProperties_url')]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
@ -264,7 +289,9 @@
},
"schema": []
},
"dependsOn": []
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]",
@ -303,7 +330,9 @@
}
]
},
"dependsOn": []
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFiles')]",
@ -346,7 +375,9 @@
},
"schema": []
},
"dependsOn": []
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]",
@ -385,7 +416,9 @@
}
}
},
"dependsOn": []
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]",
@ -802,153 +835,6 @@
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopFolders')]",
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
}
]
}

143
TrainingFactoryDev/linkedTemplates/ArmTemplate_2.json
View File

@ -12,6 +12,149 @@
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]",
"type": "Microsoft.DataFactory/factories/pipelines",

6
TrainingFactoryDev/linkedTemplates/ArmTemplate_master.json
View File

@ -170,9 +170,6 @@
},
"trainingdb01_properties_typeProperties_connectionString_secretName": {
"value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
},
"traininglake01_properties_typeProperties_url": {
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
}
}
},
@ -192,6 +189,9 @@
"factoryName": {
"value": "[parameters('factoryName')]"
},
"traininglake01_properties_typeProperties_url": {
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
},
"AzureKeyVault1_properties_privateLinkResourceId": {
"value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]"
},