be14d332c0
Adding dataflow: MappingOrderAggregation Adding dataset: LakeFileOrderHeaderParquet Adding dataset: LakeFileOrderDetailLinesParquet Adding dataset: TableOrderSummary Adding linkedService: traininglake01noneKV Adding linkedService: trainingdb01 Adding linkedService: TrainingKeys01 Adding dataflow: WranglingOrderAggregation Adding dataset: GetSetMetadata Adding dataset: AnyDatabaseTable Adding linkedService: AnyDatabaseConnection Adding dataset: LaptopFolders Adding linkedService: LaptopFiles Adding integrationRuntime: PaulsLaptopIR Adding dataset: LaptopFilePersonCSV Adding dataset: LakeFiles Adding linkedService: traininglake01 Adding dataset: LaptopsFiles Adding dataset: LakeFilePersonCSV Adding dataset: LakeFileParquet Adding integrationRuntime: ForDataFlowDemos Adding integrationRuntime: VNetEnabledIR Adding managedVirtualNetwork: default Adding linkedService: traininglak01 Adding linkedService: TrainingKeys01withUMI Adding linkedService: TrainingStore01 Adding linkedService: BatchForTraining01 Adding pipeline: Scale Out Level 2 Adding pipeline: Scale Out Level 1 Adding pipeline: Lazy Replication Adding pipeline: 03 - Upload - From Discovery Adding pipeline: 01 - Upload - Simple Adding pipeline: 04 - Upload - From Metadata Adding pipeline: Order Summary with Mapping Adding pipeline: 02 - Upload - Copy Params Adding pipeline: 1950 Activities Adding pipeline: Custom Activity Adding managedPrivateEndpoint: AzureKeyVault1 Adding factory: TrainingFactoryDev
179 lines
3.9 KiB
JSON
179 lines
3.9 KiB
JSON
{
|
|
"name": "Scale Out Level 1",
|
|
"properties": {
|
|
"activities": [
|
|
{
|
|
"name": "Get Bucket Ids",
|
|
"type": "Lookup",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "Reset And Log Start",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
},
|
|
{
|
|
"activity": "Reset Bucket Process Map",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "AzureSqlSource",
|
|
"sqlReaderStoredProcedureName": "[dbo].[GetBuckets]",
|
|
"partitionOption": "None"
|
|
},
|
|
"dataset": {
|
|
"referenceName": "GetSetMetadata",
|
|
"type": "DatasetReference"
|
|
},
|
|
"firstRowOnly": false
|
|
}
|
|
},
|
|
{
|
|
"name": "Call Buckets",
|
|
"type": "ForEach",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "Get Bucket Ids",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
}
|
|
],
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"items": {
|
|
"value": "@activity('Get Bucket Ids').output.value",
|
|
"type": "Expression"
|
|
},
|
|
"isSequential": false,
|
|
"batchCount": 50,
|
|
"activities": [
|
|
{
|
|
"name": "Call Level 2",
|
|
"type": "ExecutePipeline",
|
|
"dependsOn": [],
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"pipeline": {
|
|
"referenceName": "Scale Out Level 2",
|
|
"type": "PipelineReference"
|
|
},
|
|
"waitOnCompletion": true,
|
|
"parameters": {
|
|
"BucketId": {
|
|
"value": "@{item().BucketId}",
|
|
"type": "Expression"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "Log End",
|
|
"type": "SqlServerStoredProcedure",
|
|
"dependsOn": [
|
|
{
|
|
"activity": "Call Buckets",
|
|
"dependencyConditions": [
|
|
"Succeeded"
|
|
]
|
|
},
|
|
{
|
|
"activity": "Reset And Log Start",
|
|
"dependencyConditions": [
|
|
"Completed"
|
|
]
|
|
}
|
|
],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"storedProcedureName": "[dbo].[SetBucketLogEntry]",
|
|
"storedProcedureParameters": {
|
|
"LogId": {
|
|
"value": {
|
|
"value": "@activity('Reset And Log Start').output.firstRow.LogId",
|
|
"type": "Expression"
|
|
},
|
|
"type": "Int32"
|
|
}
|
|
}
|
|
},
|
|
"linkedServiceName": {
|
|
"referenceName": "trainingdb01",
|
|
"type": "LinkedServiceReference"
|
|
}
|
|
},
|
|
{
|
|
"name": "Reset And Log Start",
|
|
"type": "Lookup",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"source": {
|
|
"type": "AzureSqlSource",
|
|
"sqlReaderStoredProcedureName": "[dbo].[SetBucketLogEntry]",
|
|
"partitionOption": "None"
|
|
},
|
|
"dataset": {
|
|
"referenceName": "GetSetMetadata",
|
|
"type": "DatasetReference"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "Reset Bucket Process Map",
|
|
"type": "SqlServerStoredProcedure",
|
|
"dependsOn": [],
|
|
"policy": {
|
|
"timeout": "7.00:00:00",
|
|
"retry": 0,
|
|
"retryIntervalInSeconds": 30,
|
|
"secureOutput": false,
|
|
"secureInput": false
|
|
},
|
|
"userProperties": [],
|
|
"typeProperties": {
|
|
"storedProcedureName": "[dbo].[SetBucketProcesses]"
|
|
},
|
|
"linkedServiceName": {
|
|
"referenceName": "trainingdb01",
|
|
"type": "LinkedServiceReference"
|
|
}
|
|
}
|
|
],
|
|
"folder": {
|
|
"name": "Demo Pipelines/Scaling Out"
|
|
},
|
|
"annotations": [],
|
|
"lastPublishTime": "2020-08-24T13:41:19Z"
|
|
},
|
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
|
} |