954 lines
23 KiB
JSON
954 lines
23 KiB
JSON
{
|
|
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"parameters": {
|
|
"factoryName": {
|
|
"type": "string",
|
|
"metadata": "Data Factory name",
|
|
"defaultValue": "TrainingFactoryDev"
|
|
},
|
|
"AzureKeyVault1_properties_privateLinkResourceId": {
|
|
"type": "string",
|
|
"defaultValue": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01"
|
|
},
|
|
"AzureKeyVault1_properties_groupId": {
|
|
"type": "string",
|
|
"defaultValue": "vault"
|
|
},
|
|
"AzureKeyVault1_properties_fqdns": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"trainingkeys01.vaultcore.azure.net"
|
|
]
|
|
}
|
|
},
|
|
"variables": {
|
|
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/VNetEnabledIR')]",
|
|
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"type": "Managed",
|
|
"typeProperties": {
|
|
"computeProperties": {
|
|
"location": "UK South",
|
|
"dataFlowProperties": {
|
|
"computeType": "General",
|
|
"coreCount": 8,
|
|
"timeToLive": 0,
|
|
"cleanup": true
|
|
}
|
|
}
|
|
},
|
|
"managedVirtualNetwork": {
|
|
"type": "ManagedVirtualNetworkReference",
|
|
"referenceName": "default"
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/default/AzureKeyVault1')]",
|
|
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"privateLinkResourceId": "[parameters('AzureKeyVault1_properties_privateLinkResourceId')]",
|
|
"groupId": "[parameters('AzureKeyVault1_properties_groupId')]",
|
|
"fqdns": "[parameters('AzureKeyVault1_properties_fqdns')]"
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/Custom Activity')]",
|
|
"type": "Microsoft.DataFactory/factories/pipelines",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"activities": [
|
|
{
|
|
"name": "HelloWorld",
|
|
"type": "Custom",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"command": "CustomActivity.exe",
|
|
"resourceLinkedService": {
|
|
"referenceName": "TrainingStore01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"folderPath": "customactivities",
|
|
"referenceObjects": {
|
|
"linkedServices": [],
|
|
"datasets": []
|
|
}
|
|
},
|
|
"linkedServiceName": {
|
|
"referenceName": "BatchForTraining01",
|
|
"type": "LinkedServiceReference"
|
|
}
|
|
}
|
|
],
|
|
"policy": {
|
|
"elapsedTimeMetric": {},
|
|
"cancelAfter": {}
|
|
},
|
|
"folder": {
|
|
"name": "Demo Pipelines/Misc"
|
|
},
|
|
"annotations": [],
|
|
"lastPublishTime": "2022-02-03T14:39:04Z"
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTable')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "AnyDatabaseConnection",
|
|
"type": "LinkedServiceReference",
|
|
"parameters": {
|
|
"DBConnectionSecret": {
|
|
"value": "@dataset().LinkedServiceConnectionSecret",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
},
|
|
"parameters": {
|
|
"LinkedServiceConnectionSecret": {
|
|
"type": "string"
|
|
},
|
|
"SchemaName": {
|
|
"type": "string"
|
|
},
|
|
"TableName": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "SQLDB"
|
|
},
|
|
"annotations": [],
|
|
"type": "AzureSqlTable",
|
|
"schema": [],
|
|
"typeProperties": {
|
|
"schema": {
|
|
"value": "@dataset().SchemaName",
|
|
"type": "Expression"
|
|
},
|
|
"table": {
|
|
"value": "@dataset().TableName",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/AnyDatabaseTableAnyKeyVault')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
|
|
"type": "LinkedServiceReference",
|
|
"parameters": {
|
|
"KeyVaultURL": {
|
|
"value": "@dataset().KeyVaultURL",
|
|
"type": "Expression"
|
|
},
|
|
"KeyVaultSecretName": {
|
|
"value": "@dataset().LinkedServiceConnectionSecret",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
},
|
|
"parameters": {
|
|
"LinkedServiceConnectionSecret": {
|
|
"type": "string"
|
|
},
|
|
"SchemaName": {
|
|
"type": "string"
|
|
},
|
|
"TableName": {
|
|
"type": "string"
|
|
},
|
|
"KeyVaultURL": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "SQLDB"
|
|
},
|
|
"annotations": [],
|
|
"type": "AzureSqlTable",
|
|
"schema": [],
|
|
"typeProperties": {
|
|
"schema": {
|
|
"value": "@dataset().SchemaName",
|
|
"type": "Expression"
|
|
},
|
|
"table": {
|
|
"value": "@dataset().TableName",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/GetSetMetadata')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "trainingdb01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"folder": {
|
|
"name": "SQLDB"
|
|
},
|
|
"annotations": [],
|
|
"type": "AzureSqlTable",
|
|
"schema": [],
|
|
"typeProperties": {}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LakeFileParquet')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "traininglake01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
},
|
|
"File": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Lake"
|
|
},
|
|
"annotations": [],
|
|
"type": "Parquet",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "AzureBlobFSLocation",
|
|
"fileName": {
|
|
"value": "@dataset().File",
|
|
"type": "Expression"
|
|
},
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
},
|
|
"fileSystem": "datawarehouse"
|
|
},
|
|
"compressionCodec": "snappy"
|
|
},
|
|
"schema": []
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LakeFilePersonCSV')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "traininglake01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"folder": {
|
|
"name": "Lake"
|
|
},
|
|
"annotations": [],
|
|
"type": "DelimitedText",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "AzureBlobFSLocation",
|
|
"fileName": "Person.csv",
|
|
"folderPath": "Landing",
|
|
"fileSystem": "datawarehouse"
|
|
},
|
|
"columnDelimiter": ",",
|
|
"escapeChar": "\\",
|
|
"quoteChar": "\""
|
|
},
|
|
"schema": [
|
|
{
|
|
"type": "String"
|
|
},
|
|
{
|
|
"type": "String"
|
|
},
|
|
{
|
|
"type": "String"
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LakeFiles')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "traininglake01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
},
|
|
"File": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Lake"
|
|
},
|
|
"annotations": [],
|
|
"type": "DelimitedText",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "AzureBlobFSLocation",
|
|
"fileName": {
|
|
"value": "@dataset().File",
|
|
"type": "Expression"
|
|
},
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
},
|
|
"fileSystem": "datawarehouse"
|
|
},
|
|
"columnDelimiter": ",",
|
|
"escapeChar": "\\",
|
|
"quoteChar": "\""
|
|
},
|
|
"schema": []
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LakeFilesBinary')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "traininglake01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
},
|
|
"File": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Lake"
|
|
},
|
|
"annotations": [],
|
|
"type": "Binary",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "AzureBlobFSLocation",
|
|
"fileName": {
|
|
"value": "@dataset().File",
|
|
"type": "Expression"
|
|
},
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
},
|
|
"fileSystem": "datawarehouse"
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LaptopFilePersonCSV')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "LaptopFiles",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"folder": {
|
|
"name": "Laptop"
|
|
},
|
|
"annotations": [],
|
|
"type": "DelimitedText",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "FileServerLocation",
|
|
"fileName": "Person.csv",
|
|
"folderPath": "ForUpload/People"
|
|
},
|
|
"columnDelimiter": ",",
|
|
"escapeChar": "\\",
|
|
"quoteChar": "\""
|
|
},
|
|
"schema": [
|
|
{
|
|
"type": "String"
|
|
},
|
|
{
|
|
"type": "String"
|
|
},
|
|
{
|
|
"type": "String"
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LaptopFilesBinary')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "LaptopFiles",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
},
|
|
"File": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Laptop"
|
|
},
|
|
"annotations": [],
|
|
"type": "Binary",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "FileServerLocation",
|
|
"fileName": {
|
|
"value": "@dataset().File",
|
|
"type": "Expression"
|
|
},
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LaptopFilesParquet')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "LaptopFiles",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
},
|
|
"File": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Laptop"
|
|
},
|
|
"annotations": [],
|
|
"type": "Parquet",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "FileServerLocation",
|
|
"fileName": {
|
|
"value": "@dataset().File",
|
|
"type": "Expression"
|
|
},
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
}
|
|
},
|
|
"compressionCodec": "snappy"
|
|
},
|
|
"schema": []
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LaptopFolders')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "LaptopFiles",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Laptop"
|
|
},
|
|
"annotations": [],
|
|
"type": "DelimitedText",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "FileServerLocation",
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
}
|
|
},
|
|
"columnDelimiter": ",",
|
|
"escapeChar": "\\",
|
|
"quoteChar": "\""
|
|
},
|
|
"schema": []
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/LaptopsFiles')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "LaptopFiles",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"parameters": {
|
|
"Directory": {
|
|
"type": "string"
|
|
},
|
|
"File": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"folder": {
|
|
"name": "Laptop"
|
|
},
|
|
"annotations": [],
|
|
"type": "DelimitedText",
|
|
"typeProperties": {
|
|
"location": {
|
|
"type": "FileServerLocation",
|
|
"fileName": {
|
|
"value": "@dataset().File",
|
|
"type": "Expression"
|
|
},
|
|
"folderPath": {
|
|
"value": "@dataset().Directory",
|
|
"type": "Expression"
|
|
}
|
|
},
|
|
"columnDelimiter": ",",
|
|
"escapeChar": "\\",
|
|
"quoteChar": "\""
|
|
},
|
|
"schema": []
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/TableOrderSummary')]",
|
|
"type": "Microsoft.DataFactory/factories/datasets",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"linkedServiceName": {
|
|
"referenceName": "trainingdb01",
|
|
"type": "LinkedServiceReference"
|
|
},
|
|
"folder": {
|
|
"name": "SQLDB"
|
|
},
|
|
"annotations": [],
|
|
"type": "AzureSqlTable",
|
|
"schema": [
|
|
{
|
|
"name": "SalesOrderNumber",
|
|
"type": "varchar"
|
|
},
|
|
{
|
|
"name": "RecordCount",
|
|
"type": "int",
|
|
"precision": 10
|
|
}
|
|
],
|
|
"typeProperties": {
|
|
"schema": "dbo",
|
|
"table": "OrderSummary"
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/WranglingOrderAggregation')]",
|
|
"type": "Microsoft.DataFactory/factories/dataflows",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"type": "WranglingDataFlow",
|
|
"typeProperties": {
|
|
"sources": [
|
|
{
|
|
"name": "LakeFileOrderDetailLinesParquet",
|
|
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
|
|
"dataset": {
|
|
"referenceName": "LakeFileOrderDetailLinesParquet",
|
|
"type": "DatasetReference"
|
|
}
|
|
}
|
|
],
|
|
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
|
|
}
|
|
},
|
|
"dependsOn": []
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/01 - Upload - Simple')]",
|
|
"type": "Microsoft.DataFactory/factories/pipelines",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"activities": [
|
|
{
|
|
"name": "Upload Person",
|
|
"description": "hfdhgfdhfd",
|
|
"type": "Copy",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "DelimitedTextSource",
|
|
"storeSettings": {
|
|
"type": "FileServerReadSettings",
|
|
"recursive": true
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextReadSettings"
|
|
}
|
|
},
|
|
"sink": {
|
|
"type": "DelimitedTextSink",
|
|
"storeSettings": {
|
|
"type": "AzureBlobFSWriteSettings"
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextWriteSettings",
|
|
"quoteAllText": true,
|
|
"fileExtension": ".txt"
|
|
}
|
|
},
|
|
"enableStaging": false,
|
|
"translator": {
|
|
"type": "TabularTranslator",
|
|
"typeConversion": true,
|
|
"typeConversionSettings": {
|
|
"allowDataTruncation": true,
|
|
"treatBooleanAsNumber": false
|
|
}
|
|
}
|
|
},
|
|
"inputs": [
|
|
{
|
|
"referenceName": "LaptopFilePersonCSV",
|
|
"type": "DatasetReference",
|
|
"parameters": {}
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"referenceName": "LakeFilePersonCSV",
|
|
"type": "DatasetReference",
|
|
"parameters": {}
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"elapsedTimeMetric": {},
|
|
"cancelAfter": {}
|
|
},
|
|
"folder": {
|
|
"name": "Demo Pipelines/Data Uploads"
|
|
},
|
|
"annotations": [],
|
|
"lastPublishTime": "2020-09-29T13:50:28Z"
|
|
},
|
|
"dependsOn": [
|
|
"[concat(variables('factoryId'), '/datasets/LaptopFilePersonCSV')]",
|
|
"[concat(variables('factoryId'), '/datasets/LakeFilePersonCSV')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/02 - Upload - Copy Params')]",
|
|
"type": "Microsoft.DataFactory/factories/pipelines",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"activities": [
|
|
{
|
|
"name": "Upload Any File",
|
|
"type": "Copy",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "DelimitedTextSource",
|
|
"storeSettings": {
|
|
"type": "FileServerReadSettings",
|
|
"recursive": true
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextReadSettings"
|
|
}
|
|
},
|
|
"sink": {
|
|
"type": "DelimitedTextSink",
|
|
"storeSettings": {
|
|
"type": "AzureBlobFSWriteSettings"
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextWriteSettings",
|
|
"quoteAllText": true,
|
|
"fileExtension": ".txt"
|
|
}
|
|
},
|
|
"enableStaging": false,
|
|
"translator": {
|
|
"type": "TabularTranslator",
|
|
"typeConversion": true,
|
|
"typeConversionSettings": {
|
|
"allowDataTruncation": true,
|
|
"treatBooleanAsNumber": false
|
|
}
|
|
}
|
|
},
|
|
"inputs": [
|
|
{
|
|
"referenceName": "LaptopsFiles",
|
|
"type": "DatasetReference",
|
|
"parameters": {
|
|
"Directory": "ForUpload\\People",
|
|
"File": "Person.csv"
|
|
}
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"referenceName": "LakeFiles",
|
|
"type": "DatasetReference",
|
|
"parameters": {
|
|
"Directory": "Landing",
|
|
"File": "Person.csv"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"elapsedTimeMetric": {},
|
|
"cancelAfter": {}
|
|
},
|
|
"folder": {
|
|
"name": "Demo Pipelines/Data Uploads"
|
|
},
|
|
"annotations": [],
|
|
"lastPublishTime": "2020-08-27T09:06:46Z"
|
|
},
|
|
"dependsOn": [
|
|
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
|
|
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
|
|
]
|
|
},
|
|
{
|
|
"name": "[concat(parameters('factoryName'), '/03 - Upload - From Discovery')]",
|
|
"type": "Microsoft.DataFactory/factories/pipelines",
|
|
"apiVersion": "2018-06-01",
|
|
"properties": {
|
|
"activities": [
|
|
{
|
|
"name": "Get File List",
|
|
"type": "GetMetadata",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"dataset": {
|
|
"referenceName": "LaptopFolders",
|
|
"type": "DatasetReference",
|
|
"parameters": {
|
|
"Directory": "ForUpload\\People"
|
|
}
|
|
},
|
|
"fieldList": [
|
|
"childItems"
|
|
],
|
|
"storeSettings": {
|
|
"type": "FileServerReadSettings",
|
|
"recursive": true
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextReadSettings"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "Upload Files",
|
|
"type": "ForEach",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "Get File List",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"items": {
|
|
"value": "@activity('Get File List').output.childItems",
|
|
"type": "Expression"
|
|
},
|
|
"activities": [
|
|
{
|
|
"name": "Upload File",
|
|
"type": "Copy",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "DelimitedTextSource",
|
|
"storeSettings": {
|
|
"type": "FileServerReadSettings",
|
|
"recursive": true
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextReadSettings"
|
|
}
|
|
},
|
|
"sink": {
|
|
"type": "DelimitedTextSink",
|
|
"storeSettings": {
|
|
"type": "AzureBlobFSWriteSettings"
|
|
},
|
|
"formatSettings": {
|
|
"type": "DelimitedTextWriteSettings",
|
|
"quoteAllText": true,
|
|
"fileExtension": ".txt"
|
|
}
|
|
},
|
|
"enableStaging": false,
|
|
"translator": {
|
|
"type": "TabularTranslator",
|
|
"typeConversion": true,
|
|
"typeConversionSettings": {
|
|
"allowDataTruncation": true,
|
|
"treatBooleanAsNumber": false
|
|
}
|
|
}
|
|
},
|
|
"inputs": [
|
|
{
|
|
"referenceName": "LaptopsFiles",
|
|
"type": "DatasetReference",
|
|
"parameters": {
|
|
"Directory": "ForUpload\\People",
|
|
"File": {
|
|
"value": "@item().name",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"referenceName": "LakeFiles",
|
|
"type": "DatasetReference",
|
|
"parameters": {
|
|
"Directory": "Landing",
|
|
"File": {
|
|
"value": "@item().name",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"policy": {
|
|
"elapsedTimeMetric": {},
|
|
"cancelAfter": {}
|
|
},
|
|
"folder": {
|
|
"name": "Demo Pipelines/Data Uploads"
|
|
},
|
|
"annotations": [],
|
|
"lastPublishTime": "2020-08-27T09:06:45Z"
|
|
},
|
|
"dependsOn": [
|
|
"[concat(variables('factoryId'), '/datasets/LaptopFolders')]",
|
|
"[concat(variables('factoryId'), '/datasets/LaptopsFiles')]",
|
|
"[concat(variables('factoryId'), '/datasets/LakeFiles')]"
|
|
]
|
|
}
|
|
]
|
|
} |