dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3 from mrpaulandrewltd/one-day-labs-update

Browse Source 
Revised labs
This commit is contained in:
Paul Andrew 2023-06-07 20:46:03 +01:00 committed by GitHub
commit 266848a437
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 718 additions and 1082 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
Labs/Lab 1 - Create Azure resources.pdf
View File

Binary file not shown.

BIN
Labs/Lab 2 - Build a copy pipeline.pdf
View File

Binary file not shown.

BIN
Labs/Lab 3 - Create a reusable pipeline.pdf
View File

Binary file not shown.

BIN
Labs/Lab 4 - Author a data flow.pdf
View File

Binary file not shown.

BIN
Labs/Lab 5 - Integrate on-premises data.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 5 - Monitor factory activity.pdf
View File

Binary file not shown.

BIN
Labs/Lab 6 - Explore Synapse pipelines.pdf
View File

Binary file not shown.

BIN
Labs/Lab 6 - Monitor factory activity.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 7 - Mini-project.pdf
View File

Binary file not shown.

1082
Labs/TableCatalog.json
View File

File diff suppressed because it is too large Load Diff

175
Labs/adf/dataflow/UpdateProductDimension.json Normal file
View File

@ -0,0 +1,175 @@
{
"name": "UpdateProductDimension",
"properties": {
"folder": {
"name": "Lab4"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake",
"description": "Write results to data lake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectProductSubcategoryColumns"
},
{
"name": "SelectProductCategoryColumns"
},
{
"name": "LookupProductCategory",
"description": "Lookup product category name"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as short,",
" Subcategory as integer,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns",
"SelectProductSubcategoryColumns, SelectProductCategoryColumns lookup(SelectProductSubcategoryColumns@CategoryId == SelectProductCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectProductSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectProductSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectProductSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectProductCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'delta',",
" fileSystem: 'lakeroot',",
" folderPath: 'Prepared/DimProduct',",
" overwrite:true,",
" mergeSchema: false,",
" autoCompact: false,",
" optimizedWrite: false,",
" vacuum: 0,",
" deletable:false,",
" insertable:true,",
" updateable:false,",
" upsertable:false,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}

23
Labs/adf/dataset/ADLS_BIN_AWProduct.json Normal file
View File

@ -0,0 +1,23 @@
{
"name": "ADLS_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab2"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Product.csv",
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

31
Labs/adf/dataset/ADLS_BIN_AdventureWorks.json Normal file
View File

@ -0,0 +1,31 @@
{
"name": "ADLS_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Lab3"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().Entity}.tsv",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

22
Labs/adf/dataset/ADLS_MyLocalFile.json Normal file
View File

@ -0,0 +1,22 @@
{
"name": "ADLS_MyLocalFile",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab5"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "mylocalfile.txt",
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}

35
Labs/adf/dataset/ADLS_TSV_AdventureWorks.json Normal file
View File

@ -0,0 +1,35 @@
{
"name": "ADLS_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"parameters": {
"FileName": {
"type": "string"
}
},
"folder": {
"name": "Lab4"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().FileName",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

21
Labs/adf/dataset/FS_MyFiles.json Normal file
View File

@ -0,0 +1,21 @@
{
"name": "FS_MyFiles",
"properties": {
"linkedServiceName": {
"referenceName": "FS_MyLaptop",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab5"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "data-file.txt",
"folderPath": "tmp"
}
}
}
}

20
Labs/adf/dataset/HTTP_BIN_AWProduct.json Normal file
View File

@ -0,0 +1,20 @@
{
"name": "HTTP_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWProduct",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab2"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

29
Labs/adf/dataset/HTTP_BIN_AdventureWorks.json Normal file
View File

@ -0,0 +1,29 @@
{
"name": "HTTP_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Lab3"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().Entity}.csv",
"type": "Expression"
}
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

9
Labs/adf/factory/adflabs2023-adf.json Normal file
View File

@ -0,0 +1,9 @@
{
"name": "adflabs2023-adf",
"location": "uksouth",
"identity": {
"type": "SystemAssigned",
"principalId": "b9cd3fd6-9668-41f7-ae62-413b409b5f58",
"tenantId": "2cdc9c7e-334f-4f88-9de8-a844761a49b6"
}
}

6
Labs/adf/integrationRuntime/SHIR.json Normal file
View File

@ -0,0 +1,6 @@
{
"name": "SHIR",
"properties": {
"type": "SelfHosted"
}
}

11
Labs/adf/linkedService/AzureDataLakeStorage1.json Normal file
View File

@ -0,0 +1,11 @@
{
"name": "AzureDataLakeStorage1",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://adflabs2023sa.dfs.core.windows.net/"
}
}
}

17
Labs/adf/linkedService/FS_MyLaptop.json Normal file
View File

@ -0,0 +1,17 @@
{
"name": "FS_MyLaptop",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "FileServer",
"typeProperties": {
"host": "C:\\",
"userId": "richard",
"encryptedCredential": "eyJDcmVkZW50aWFsSWQiOiIzZDk3NmY1My0zNGYzLTQ1NWUtYjE0MS01ZGMyYmI0MmY2YmQiLCJWZXJzaW9uIjoiMi4wIiwiQ2xhc3NUeXBlIjoiTWljcm9zb2Z0LkRhdGFQcm94eS5Db3JlLkludGVyU2VydmljZURhdGFDb250cmFjdC5DcmVkZW50aWFsU1UwNkNZMTQifQ=="
},
"connectVia": {
"referenceName": "SHIR",
"type": "IntegrationRuntimeReference"
}
}
}

13
Labs/adf/linkedService/HTTP_AWGitHub.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "HTTP_AWGitHub",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}

13
Labs/adf/linkedService/HTTP_AWProduct.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "HTTP_AWProduct",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}

48
Labs/adf/pipeline/BuildDimProduct.json Normal file
View File

@ -0,0 +1,48 @@
{
"name": "BuildDimProduct",
"properties": {
"activities": [
{
"name": "Run UpdateProductDimension",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "UpdateProductDimension",
"type": "DataFlowReference",
"datasetParameters": {
"Product": {
"FileName": "Product.tsv"
},
"ProductSubcategory": {
"FileName": "ProductSubcategory.tsv"
},
"ProductCategory": {
"FileName": "ProductCategory.tsv"
}
}
},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
}
],
"folder": {
"name": "Lab4"
},
"annotations": [],
"lastPublishTime": "2023-05-28T17:33:03Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

56
Labs/adf/pipeline/CopyAWEntities.json Normal file
View File

@ -0,0 +1,56 @@
{
"name": "CopyAWEntities",
"properties": {
"activities": [
{
"name": "Copy each AW entity",
"type": "ForEach",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@variables('Entities')",
"type": "Expression"
},
"activities": [
{
"name": "Execute CopyAWEntity",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "CopyAWEntity",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Entity": {
"value": "@item()",
"type": "Expression"
}
}
}
}
]
}
}
],
"variables": {
"Entities": {
"type": "Array",
"defaultValue": [
"Product",
"ProductSubcategory",
"ProductCategory"
]
}
},
"folder": {
"name": "Lab3"
},
"annotations": [],
"lastPublishTime": "2023-05-28T14:23:19Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

74
Labs/adf/pipeline/CopyAWEntity.json Normal file
View File

@ -0,0 +1,74 @@
{
"name": "CopyAWEntity",
"properties": {
"activities": [
{
"name": "Copy AW entity",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
]
}
],
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Lab3"
},
"annotations": [],
"lastPublishTime": "2023-05-28T14:05:42Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

57
Labs/adf/pipeline/CopyAWProduct.json Normal file
View File

@ -0,0 +1,57 @@
{
"name": "CopyAWProduct",
"properties": {
"activities": [
{
"name": "Copy AW Products to data lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AWProduct",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AWProduct",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab2"
},
"annotations": [],
"lastPublishTime": "2023-05-27T20:00:12Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

57
Labs/adf/pipeline/CopyLocalFile.json Normal file
View File

@ -0,0 +1,57 @@
{
"name": "CopyLocalFile",
"properties": {
"activities": [
{
"name": "Copy AW Products to data lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "FS_MyFiles",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_MyLocalFile",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab5"
},
"annotations": [],
"lastPublishTime": "2023-05-27T20:00:12Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

1
Labs/adf/publish_config.json Normal file
View File

@ -0,0 +1 @@
{"publishBranch":"adf_publish","enableGitComment":true}