dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revised labs

Browse Source
This commit is contained in:
Richard Swinbank
2023-05-31 09:09:28 +01:00
parent 7d00f1dcde
commit b8984ccca5
30 changed files with 718 additions and 1082 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
Labs/adf/pipeline/BuildDimProduct.json Normal file
View File

@@ -0,0 +1,48 @@
{
"name": "BuildDimProduct",
"properties": {
"activities": [
{
"name": "Run UpdateProductDimension",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "UpdateProductDimension",
"type": "DataFlowReference",
"datasetParameters": {
"Product": {
"FileName": "Product.tsv"
},
"ProductSubcategory": {
"FileName": "ProductSubcategory.tsv"
},
"ProductCategory": {
"FileName": "ProductCategory.tsv"
}
}
},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
}
],
"folder": {
"name": "Lab4"
},
"annotations": [],
"lastPublishTime": "2023-05-28T17:33:03Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

56
Labs/adf/pipeline/CopyAWEntities.json Normal file
View File

@@ -0,0 +1,56 @@
{
"name": "CopyAWEntities",
"properties": {
"activities": [
{
"name": "Copy each AW entity",
"type": "ForEach",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@variables('Entities')",
"type": "Expression"
},
"activities": [
{
"name": "Execute CopyAWEntity",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "CopyAWEntity",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Entity": {
"value": "@item()",
"type": "Expression"
}
}
}
}
]
}
}
],
"variables": {
"Entities": {
"type": "Array",
"defaultValue": [
"Product",
"ProductSubcategory",
"ProductCategory"
]
}
},
"folder": {
"name": "Lab3"
},
"annotations": [],
"lastPublishTime": "2023-05-28T14:23:19Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

74
Labs/adf/pipeline/CopyAWEntity.json Normal file
View File

@@ -0,0 +1,74 @@
{
"name": "CopyAWEntity",
"properties": {
"activities": [
{
"name": "Copy AW entity",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
]
}
],
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Lab3"
},
"annotations": [],
"lastPublishTime": "2023-05-28T14:05:42Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

57
Labs/adf/pipeline/CopyAWProduct.json Normal file
View File

@@ -0,0 +1,57 @@
{
"name": "CopyAWProduct",
"properties": {
"activities": [
{
"name": "Copy AW Products to data lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AWProduct",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AWProduct",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab2"
},
"annotations": [],
"lastPublishTime": "2023-05-27T20:00:12Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

57
Labs/adf/pipeline/CopyLocalFile.json Normal file
View File

@@ -0,0 +1,57 @@
{
"name": "CopyLocalFile",
"properties": {
"activities": [
{
"name": "Copy AW Products to data lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "FS_MyFiles",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_MyLocalFile",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab5"
},
"annotations": [],
"lastPublishTime": "2023-05-27T20:00:12Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}