ARM template and parameters deployed on 3-9-2022 15:59:37, based on the collaboration branch's commit ID: 09c848bc7d65f8fe7a0f7efad419136c1941ef5d

This commit is contained in:
Paul Andrew 2022-03-09 15:59:39 +00:00
parent 104531fb1f
commit 2a23b67be2
5 changed files with 231 additions and 181 deletions

View File

@ -3259,6 +3259,30 @@
"dependsOn": [ "dependsOn": [
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]" "[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
] ]
},
{
"name": "[concat(parameters('factoryName'), '/pipeline1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"annotations": []
},
"dependsOn": []
} }
] ]
} }

View File

@ -74,10 +74,6 @@
"trainingdb01_properties_typeProperties_connectionString_secretName": { "trainingdb01_properties_typeProperties_connectionString_secretName": {
"type": "string", "type": "string",
"defaultValue": "ConnectionString-trainingdb01" "defaultValue": "ConnectionString-trainingdb01"
},
"traininglake01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
} }
}, },
"variables": { "variables": {
@ -628,6 +624,30 @@
"properties": {}, "properties": {},
"dependsOn": [] "dependsOn": []
}, },
{
"name": "[concat(parameters('factoryName'), '/pipeline1')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Wait1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 1
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"annotations": []
},
"dependsOn": []
},
{ {
"name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]", "name": "[concat(parameters('factoryName'), '/LakeFileOrderDetailLinesParquet')]",
"type": "Microsoft.DataFactory/factories/datasets", "type": "Microsoft.DataFactory/factories/datasets",
@ -985,29 +1005,6 @@
"dependsOn": [ "dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]" "[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
] ]
},
{
"name": "[concat(parameters('factoryName'), '/traininglake01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglake01_properties_typeProperties_url')]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/TrainingKeys01')]"
]
} }
] ]
} }

View File

@ -7,6 +7,10 @@
"metadata": "Data Factory name", "metadata": "Data Factory name",
"defaultValue": "TrainingFactoryDev" "defaultValue": "TrainingFactoryDev"
}, },
"traininglake01_properties_typeProperties_url": {
"type": "string",
"defaultValue": "https://traininglake01.dfs.core.windows.net"
},
"AzureKeyVault1_properties_privateLinkResourceId": { "AzureKeyVault1_properties_privateLinkResourceId": {
"type": "string", "type": "string",
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01" "defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
@ -26,6 +30,27 @@
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
}, },
"resources": [ "resources": [
{
"name": "[concat(parameters('factoryName'), '/traininglake01')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "[parameters('traininglake01_properties_typeProperties_url')]",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
},
"dependsOn": []
},
{ {
"name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]", "name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes", "type": "Microsoft.DataFactory/factories/integrationRuntimes",
@ -264,7 +289,9 @@
}, },
"schema": [] "schema": []
}, },
"dependsOn": [] "dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
}, },
{ {
"name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]", "name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]",
@ -303,7 +330,9 @@
} }
] ]
}, },
"dependsOn": [] "dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
}, },
{ {
"name": "[concat(parameters('factoryName'), '/LakeFiles')]", "name": "[concat(parameters('factoryName'), '/LakeFiles')]",
@ -346,7 +375,9 @@
}, },
"schema": [] "schema": []
}, },
"dependsOn": [] "dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
}, },
{ {
"name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]", "name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]",
@ -385,7 +416,9 @@
} }
} }
}, },
"dependsOn": [] "dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/traininglake01')]"
]
}, },
{ {
"name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]", "name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]",
@ -802,153 +835,6 @@
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]", "[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]" "[concat(variables('factoryId'), '/datasets/LakeFiles')]"
] ]
},
{
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/LaptopFolders')]",
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
]
} }
] ]
} }

View File

@ -12,6 +12,149 @@
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
}, },
"resources": [ "resources": [
{
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"policy": {
"elapsedTimeMetric": {},
"cancelAfter": {}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"dependsOn": []
},
{ {
"name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]", "name": "[concat(parameters('factoryName'), '/04 - Upload - From Metadata')]",
"type": "Microsoft.DataFactory/factories/pipelines", "type": "Microsoft.DataFactory/factories/pipelines",

View File

@ -170,9 +170,6 @@
}, },
"trainingdb01_properties_typeProperties_connectionString_secretName": { "trainingdb01_properties_typeProperties_connectionString_secretName": {
"value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]" "value": "[parameters('trainingdb01_properties_typeProperties_connectionString_secretName')]"
},
"traininglake01_properties_typeProperties_url": {
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
} }
} }
}, },
@ -192,6 +189,9 @@
"factoryName": { "factoryName": {
"value": "[parameters('factoryName')]" "value": "[parameters('factoryName')]"
}, },
"traininglake01_properties_typeProperties_url": {
"value": "[parameters('traininglake01_properties_typeProperties_url')]"
},
"AzureKeyVault1_properties_privateLinkResourceId": { "AzureKeyVault1_properties_privateLinkResourceId": {
"value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]" "value": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]"
}, },