Updating pipeline: CopyAndMapAWEntity

Code/DataFactory/pipeline/CopyAndMapAWEntity.json

@@ -3,8 +3,8 @@
 	"properties": {
 		"activities": [
 			{
-				"name": "Copy data1",
+				"name": "Get catalog",
-				"type": "Copy",
+				"type": "WebActivity",
 				"dependsOn": [],
 				"policy": {
 					"timeout": "0.12:00:00",
@@ -15,101 +15,116 @@
 				},
 				"userProperties": [],
 				"typeProperties": {
-					"source": {
+					"url": {
-						"type": "DelimitedTextSource",
+						"value": "https://raw.githubusercontent.com/richardswinbank/adf-metadata/main/catalog.json",
-						"storeSettings": {
+						"type": "Expression"
-							"type": "HttpReadSettings",
-							"requestMethod": "GET"
-						},
-						"formatSettings": {
-							"type": "DelimitedTextReadSettings"
-						}
 					},
-					"sink": {
+					"method": "GET"
-						"type": "ParquetSink",
+				}
-						"storeSettings": {
+			},
-							"type": "AzureBlobFSWriteSettings"
+			{
-						},
+				"name": "Cache catalog as JSON array",
-						"formatSettings": {
+				"type": "SetVariable",
-							"type": "ParquetWriteSettings"
+				"dependsOn": [
-						}
-					},
-					"enableStaging": false,
-					"translator": {
-						"type": "TabularTranslator",
-						"mappings": [
-							{
-								"source": {
-									"type": "Int32",
-									"ordinal": 1
-								},
-								"sink": {
-									"name": "ProductCategoryID",
-									"physicalType": "UTF8"
-								}
-							},
-							{
-								"source": {
-									"type": "String",
-									"ordinal": 2
-								},
-								"sink": {
-									"name": "Name",
-									"physicalType": "UTF8"
-								}
-							},
-							{
-								"source": {
-									"type": "Guid",
-									"ordinal": 3
-								},
-								"sink": {
-									"name": "rowguid",
-									"physicalType": "UTF8"
-								}
-							},
-							{
-								"source": {
-									"type": "DateTime",
-									"ordinal": 4
-								},
-								"sink": {
-									"name": "ModifiedDate",
-									"physicalType": "UTF8"
-								}
-							}
-						],
-						"typeConversion": true,
-						"typeConversionSettings": {
-							"allowDataTruncation": true,
-							"treatBooleanAsNumber": false
-						}
-					}
-				},
-				"inputs": [
 					{
-						"referenceName": "HTTP_TSV_AdventureWorks",
+						"activity": "Get catalog",
-						"type": "DatasetReference",
+						"dependencyConditions": [
-						"parameters": {
+							"Succeeded"
-							"EntityName": {
+						]
-								"value": "@pipeline().parameters.EntityName",
-								"type": "Expression"
-							}
-						}
 					}
 				],
-				"outputs": [
+				"userProperties": [],
-					{
+				"typeProperties": {
-						"referenceName": "ADLS_PQT_AdventureWorks",
+					"variableName": "Catalog",
-						"type": "DatasetReference",
+					"value": {
-						"parameters": {
+						"value": "@json(activity('Get catalog').output.Response)",
-							"EntityName": {
+						"type": "Expression"
-								"value": "@pipeline().parameters.EntityName",
-								"type": "Expression"
-							}
-						}
 					}
-				]
+				}
+			},
+			{
+				"name": "ForEach AW entity",
+				"type": "ForEach",
+				"dependsOn": [
+					{
+						"activity": "Cache catalog as JSON array",
+						"dependencyConditions": [
+							"Succeeded"
+						]
+					}
+				],
+				"userProperties": [],
+				"typeProperties": {
+					"items": {
+						"value": "@variables('Catalog')",
+						"type": "Expression"
+					},
+					"activities": [
+						{
+							"name": "Copy AW entity",
+							"type": "Copy",
+							"dependsOn": [],
+							"policy": {
+								"timeout": "0.12:00:00",
+								"retry": 0,
+								"retryIntervalInSeconds": 30,
+								"secureOutput": false,
+								"secureInput": false
+							},
+							"userProperties": [],
+							"typeProperties": {
+								"source": {
+									"type": "DelimitedTextSource",
+									"storeSettings": {
+										"type": "HttpReadSettings",
+										"requestMethod": "GET"
+									},
+									"formatSettings": {
+										"type": "DelimitedTextReadSettings"
+									}
+								},
+								"sink": {
+									"type": "ParquetSink",
+									"storeSettings": {
+										"type": "AzureBlobFSWriteSettings"
+									},
+									"formatSettings": {
+										"type": "ParquetWriteSettings"
+									}
+								},
+								"enableStaging": false,
+								"translator": {
+									"value": "@item().translator",
+									"type": "Expression"
+								}
+							},
+							"inputs": [
+								{
+									"referenceName": "HTTP_TSV_AdventureWorks",
+									"type": "DatasetReference",
+									"parameters": {
+										"EntityName": {
+											"value": "@item().EntityName",
+											"type": "Expression"
+										}
+									}
+								}
+							],
+							"outputs": [
+								{
+									"referenceName": "ADLS_PQT_AdventureWorks",
+									"type": "DatasetReference",
+									"parameters": {
+										"EntityName": {
+											"value": "@item().EntityName",
+											"type": "Expression"
+										}
+									}
+								}
+							]
+						}
+					]
+				}
 			}
 		],
 		"parameters": {
@@ -118,6 +133,12 @@
 				"defaultValue": "ProductCategory"
 			}
 		},
+		"variables": {
+			"Catalog": {
+				"type": "Array",
+				"defaultValue": []
+			}
+		},
 		"folder": {
 			"name": "Labs2"
 		},