Adding credential: TrainingCredential01

Adding dataflow: MappingOrderAggregation Adding dataset: LakeFileOrderHeaderParquet Adding dataset: LakeFileOrderDetailLinesParquet Adding dataset: TableOrderSummary Adding linkedService: traininglake01noneKV Adding linkedService: trainingdb01 Adding linkedService: TrainingKeys01 Adding dataflow: WranglingOrderAggregation Adding dataset: GetSetMetadata Adding dataset: AnyDatabaseTable Adding linkedService: AnyDatabaseConnection Adding dataset: LaptopFolders Adding linkedService: LaptopFiles Adding integrationRuntime: PaulsLaptopIR Adding dataset: LaptopFilePersonCSV Adding dataset: LakeFiles Adding linkedService: traininglake01 Adding dataset: LaptopsFiles Adding dataset: LakeFilePersonCSV Adding dataset: LakeFileParquet Adding integrationRuntime: ForDataFlowDemos Adding integrationRuntime: VNetEnabledIR Adding managedVirtualNetwork: default Adding linkedService: traininglak01 Adding linkedService: TrainingKeys01withUMI Adding linkedService: TrainingStore01 Adding linkedService: BatchForTraining01 Adding pipeline: Scale Out Level 2 Adding pipeline: Scale Out Level 1 Adding pipeline: Lazy Replication Adding pipeline: 03 - Upload - From Discovery Adding pipeline: 01 - Upload - Simple Adding pipeline: 04 - Upload - From Metadata Adding pipeline: Order Summary with Mapping Adding pipeline: 02 - Upload - Copy Params Adding pipeline: 1950 Activities Adding pipeline: Custom Activity Adding managedPrivateEndpoint: AzureKeyVault1 Adding factory: TrainingFactoryDev
2022-02-11 10:15:02 +00:00
parent 21254cf18e
commit be14d332c0
40 changed files with 2268 additions and 0 deletions
--- a/Code/DataFactory/pipeline/Scale
+++ b/Code/DataFactory/pipeline/Scale
@@ -0,0 +1,179 @@
+{
+	"name": "Scale Out Level 1",
+	"properties": {
+		"activities": [
+			{
+				"name": "Get Bucket Ids",
+				"type": "Lookup",
+				"dependsOn": [
+					{
+						"activity": "Reset And Log Start",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					},
+					{
+						"activity": "Reset Bucket Process Map",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"policy": {
+					"timeout": "7.00:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"source": {
+						"type": "AzureSqlSource",
+						"sqlReaderStoredProcedureName": "[dbo].[GetBuckets]",
+						"partitionOption": "None"
+					},
+					"dataset": {
+						"referenceName": "GetSetMetadata",
+						"type": "DatasetReference"
+					},
+					"firstRowOnly": false
+				}
+			},
+			{
+				"name": "Call Buckets",
+				"type": "ForEach",
+				"dependsOn": [
+					{
+						"activity": "Get Bucket Ids",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"userProperties": [],
+				"typeProperties": {
+					"items": {
+						"value": "@activity('Get Bucket Ids').output.value",
+						"type": "Expression"
+					},
+					"isSequential": false,
+					"batchCount": 50,
+					"activities": [
+						{
+							"name": "Call Level 2",
+							"type": "ExecutePipeline",
+							"dependsOn": [],
+							"userProperties": [],
+							"typeProperties": {
+								"pipeline": {
+									"referenceName": "Scale Out Level 2",
+									"type": "PipelineReference"
+								},
+								"waitOnCompletion": true,
+								"parameters": {
+									"BucketId": {
+										"value": "@{item().BucketId}",
+										"type": "Expression"
+									}
+								}
+							}
+						}
+					]
+				}
+			},
+			{
+				"name": "Log End",
+				"type": "SqlServerStoredProcedure",
+				"dependsOn": [
+					{
+						"activity": "Call Buckets",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					},
+					{
+						"activity": "Reset And Log Start",
+						"dependencyConditions": [
+							"Completed"
+						]
+					}
+				],
+				"policy": {
+					"timeout": "7.00:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"storedProcedureName": "[dbo].[SetBucketLogEntry]",
+					"storedProcedureParameters": {
+						"LogId": {
+							"value": {
+								"value": "@activity('Reset And Log Start').output.firstRow.LogId",
+								"type": "Expression"
+							},
+							"type": "Int32"
+						}
+					}
+				},
+				"linkedServiceName": {
+					"referenceName": "trainingdb01",
+					"type": "LinkedServiceReference"
+				}
+			},
+			{
+				"name": "Reset And Log Start",
+				"type": "Lookup",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "7.00:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"source": {
+						"type": "AzureSqlSource",
+						"sqlReaderStoredProcedureName": "[dbo].[SetBucketLogEntry]",
+						"partitionOption": "None"
+					},
+					"dataset": {
+						"referenceName": "GetSetMetadata",
+						"type": "DatasetReference"
+					}
+				}
+			},
+			{
+				"name": "Reset Bucket Process Map",
+				"type": "SqlServerStoredProcedure",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "7.00:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"storedProcedureName": "[dbo].[SetBucketProcesses]"
+				},
+				"linkedServiceName": {
+					"referenceName": "trainingdb01",
+					"type": "LinkedServiceReference"
+				}
+			}
+		],
+		"folder": {
+			"name": "Demo Pipelines/Scaling Out"
+		},
+		"annotations": [],
+		"lastPublishTime": "2020-08-24T13:41:19Z"
+	},
+	"type": "Microsoft.DataFactory/factories/pipelines"
+}