Renaming pipeline: CopyAndMapAWEntity as CopyAWUsingMetadata

2022-10-22 17:19:49 +01:00
parent 8cb67f0240
commit 608fcaff97
1 changed files with 0 additions and 0 deletions
--- a/Code/DataFactory/pipeline/CopyAWUsingMetadata.json
+++ b/Code/DataFactory/pipeline/CopyAWUsingMetadata.json
@@ -0,0 +1,176 @@
+{
+	"name": "CopyAndMapAWEntity",
+	"properties": {
+		"activities": [
+			{
+				"name": "Get catalog",
+				"type": "WebActivity",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json",
+					"method": "GET"
+				}
+			},
+			{
+				"name": "Cache catalog as JSON array",
+				"type": "SetVariable",
+				"dependsOn": [
+					{
+						"activity": "Get catalog",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"userProperties": [],
+				"typeProperties": {
+					"variableName": "Catalog",
+					"value": {
+						"value": "@json(activity('Get catalog').output.Response)",
+						"type": "Expression"
+					}
+				}
+			},
+			{
+				"name": "ForEach AW entity",
+				"type": "ForEach",
+				"dependsOn": [
+					{
+						"activity": "Cache catalog as JSON array",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"userProperties": [],
+				"typeProperties": {
+					"items": {
+						"value": "@variables('Catalog')",
+						"type": "Expression"
+					},
+					"activities": [
+						{
+							"name": "Copy AW entity",
+							"type": "Copy",
+							"dependsOn": [],
+							"policy": {
+								"timeout": "0.12:00:00",
+								"retry": 0,
+								"retryIntervalInSeconds": 30,
+								"secureOutput": false,
+								"secureInput": false
+							},
+							"userProperties": [],
+							"typeProperties": {
+								"source": {
+									"type": "DelimitedTextSource",
+									"storeSettings": {
+										"type": "HttpReadSettings",
+										"requestMethod": "GET"
+									},
+									"formatSettings": {
+										"type": "DelimitedTextReadSettings"
+									}
+								},
+								"sink": {
+									"type": "ParquetSink",
+									"storeSettings": {
+										"type": "AzureBlobFSWriteSettings"
+									},
+									"formatSettings": {
+										"type": "ParquetWriteSettings"
+									}
+								},
+								"enableStaging": false,
+								"translator": {
+									"value": "@item().translator",
+									"type": "Expression"
+								}
+							},
+							"inputs": [
+								{
+									"referenceName": "HTTP_TSV_AdventureWorks",
+									"type": "DatasetReference",
+									"parameters": {
+										"EntityName": {
+											"value": "@item().EntityName",
+											"type": "Expression"
+										}
+									}
+								}
+							],
+							"outputs": [
+								{
+									"referenceName": "ADLS_PQT_AdventureWorks",
+									"type": "DatasetReference",
+									"parameters": {
+										"EntityName": {
+											"value": "@item().EntityName",
+											"type": "Expression"
+										}
+									}
+								}
+							]
+						},
+						{
+							"name": "Notify error",
+							"type": "AzureFunctionActivity",
+							"dependsOn": [
+								{
+									"activity": "Copy AW entity",
+									"dependencyConditions": [
+										"Failed"
+									]
+								}
+							],
+							"policy": {
+								"timeout": "0.12:00:00",
+								"retry": 0,
+								"retryIntervalInSeconds": 30,
+								"secureOutput": false,
+								"secureInput": false
+							},
+							"userProperties": [],
+							"typeProperties": {
+								"functionName": "SendEmail",
+								"method": "POST",
+								"body": {
+									"value": "{\n    \"emailRecipients\": \"richard.swinbank@gmail.com\",\n    \"emailSubject\": \"Error copying @{item().EntityName}\",\n    \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}",
+									"type": "Expression"
+								}
+							},
+							"linkedServiceName": {
+								"referenceName": "EmailSenderFunction",
+								"type": "LinkedServiceReference"
+							}
+						}
+					]
+				}
+			}
+		],
+		"parameters": {
+			"EntityName": {
+				"type": "string",
+				"defaultValue": "ProductCategory"
+			}
+		},
+		"variables": {
+			"Catalog": {
+				"type": "Array",
+				"defaultValue": []
+			}
+		},
+		"folder": {
+			"name": "Labs2"
+		},
+		"annotations": []
+	}
+}