dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2 from mrpaulandrewltd/copenhagen-labs2

Browse Source 
Copenhagen labs2
This commit is contained in:
Richard Swinbank
2022-10-24 12:11:16 +01:00
committed by GitHub
parent 5e460f3eb4 8b5b6e7815
commit d839132a0c
14 changed files with 641 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Code/DataFactory/dataset/ADLS_BIN_AWProduct.json
View File

@@ -6,7 +6,7 @@
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": [],
"type": "Binary",

2
Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json
View File

@@ -11,7 +11,7 @@
}
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": [],
"type": "Binary",

32
Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json Normal file
View File

@@ -0,0 +1,32 @@
{
"name": "ADLS_PQT_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"EntityName": {
"type": "string"
}
},
"folder": {
"name": "Labs2"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().EntityName}.parquet",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"compressionCodec": "snappy"
},
"schema": []
}
}

2
Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json
View File

@@ -11,7 +11,7 @@
}
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": [],
"type": "DelimitedText",

2
Code/DataFactory/dataset/HTTP_BIN_AWProduct.json
View File

@@ -6,7 +6,7 @@
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": [],
"type": "Binary",

2
Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json
View File

@@ -11,7 +11,7 @@
}
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": [],
"type": "Binary",

32
Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json Normal file
View File

@@ -0,0 +1,32 @@
{
"name": "HTTP_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"EntityName": {
"type": "string"
}
},
"folder": {
"name": "Labs2"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().EntityName}.csv",
"type": "Expression"
}
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
}
}

2
Code/DataFactory/pipeline/BuildDimProduct.json
View File

@@ -39,7 +39,7 @@
}
],
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": []
}

2
Code/DataFactory/pipeline/CopyAWEntities.json
View File

@@ -48,7 +48,7 @@
}
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": []
}

2
Code/DataFactory/pipeline/CopyAWEntity.json
View File

@@ -65,7 +65,7 @@
}
},
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": []
}

2
Code/DataFactory/pipeline/CopyAWProduct.json
View File

@@ -48,7 +48,7 @@
}
],
"folder": {
"name": "Labs"
"name": "Labs1"
},
"annotations": []
}

170
Code/DataFactory/pipeline/CopyAWUsingMetadata.json Normal file
View File

@@ -0,0 +1,170 @@
{
"name": "CopyAWUsingMetadata",
"properties": {
"activities": [
{
"name": "Get catalog",
"type": "WebActivity",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json",
"method": "GET"
}
},
{
"name": "Cache catalog as JSON array",
"type": "SetVariable",
"dependsOn": [
{
"activity": "Get catalog",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"variableName": "Catalog",
"value": {
"value": "@json(activity('Get catalog').output.Response)",
"type": "Expression"
}
}
},
{
"name": "ForEach AW entity",
"type": "ForEach",
"dependsOn": [
{
"activity": "Cache catalog as JSON array",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@variables('Catalog')",
"type": "Expression"
},
"activities": [
{
"name": "Copy AW entity",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "ParquetSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "ParquetWriteSettings"
}
},
"enableStaging": false,
"translator": {
"value": "@item().translator",
"type": "Expression"
}
},
"inputs": [
{
"referenceName": "HTTP_TSV_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"EntityName": {
"value": "@item().EntityName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_PQT_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"EntityName": {
"value": "@item().EntityName",
"type": "Expression"
}
}
}
]
},
{
"name": "Notify error",
"type": "AzureFunctionActivity",
"dependsOn": [
{
"activity": "Copy AW entity",
"dependencyConditions": [
"Failed"
]
}
],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"functionName": "SendEmail",
"method": "POST",
"body": {
"value": "{\n \"emailRecipients\": \"richard.swinbank@gmail.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "EmailSenderFunction",
"type": "LinkedServiceReference"
}
}
]
}
}
],
"variables": {
"Catalog": {
"type": "Array",
"defaultValue": []
}
},
"folder": {
"name": "Labs2"
},
"annotations": []
}
}