dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1 from mrpaulandrewltd/lab-revisions

Browse Source 
Lab revisions
This commit is contained in:
Richard Swinbank 2022-08-30 11:15:01 +01:00 committed by GitHub
commit 8920a4857c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 710 additions and 830 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Code/DataFactory/dataflow/BuildProductDimension.json
View File

@ -1,64 +0,0 @@
{
"name": "BuildProductDimension",
"properties": {
"folder": {
"name": "Lab4"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_Product",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_ProductCategory",
"type": "DatasetReference"
},
"name": "ProductCategory"
},
{
"dataset": {
"referenceName": "ADLS_ProductModel",
"type": "DatasetReference"
},
"name": "ProductModel"
}
],
"sinks": [
{
"dataset": {
"referenceName": "ADLS_DimProduct",
"type": "DatasetReference"
},
"name": "DimProduct"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectProductCategoryColumns"
},
{
"name": "SelectProductModelColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductModelID"
},
{
"name": "RemoveDuplicateColumns"
}
],
"script": "source(output(\n\t\tProductID as short,\n\t\tName as string,\n\t\tProductNumber as string,\n\t\tColor as string,\n\t\tStandardCost as double,\n\t\tListPrice as double,\n\t\tSize as string,\n\t\tWeight as double,\n\t\tProductCategoryID as short,\n\t\tProductModelID as short,\n\t\tSellStartDate as string,\n\t\tSellEndDate as string,\n\t\tDiscontinuedDate as string,\n\t\tThumbNailPhoto as string,\n\t\tThumbnailPhotoFileName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> Product\nsource(output(\n\t\tProductCategoryID as short,\n\t\tParentProductCategoryID as string,\n\t\tName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductCategory\nsource(output(\n\t\tProductModelID as short,\n\t\tName as string,\n\t\tCatalogDescription as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductModel\nProduct select(mapColumn(\n\t\tProductID,\n\t\tProduct = Name,\n\t\tProductCategoryID,\n\t\tProductModelID\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductColumns\nProductCategory select(mapColumn(\n\t\tProductCategoryID,\n\t\tProductCategory = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns\nProductModel select(mapColumn(\n\t\tProductModelID,\n\t\tProductModel = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductModelColumns\nSelectProductColumns, SelectProductCategoryColumns lookup(SelectProductColumns@ProductCategoryID == SelectProductCategoryColumns@ProductCategoryID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductCategory\nLookupProductCategory, SelectProductModelColumns lookup(SelectProductColumns@ProductModelID == SelectProductModelColumns@ProductModelID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductModelID\nLookupProductModelID select(mapColumn(\n\t\tProductID,\n\t\tProduct,\n\t\tProductCategoryID = {SelectProductColumns@ProductCategoryID},\n\t\tProductModelID = {SelectProductColumns@ProductModelID},\n\t\tProductCategory,\n\t\tProductModel\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns\nRemoveDuplicateColumns sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'parquet',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> DimProduct"
}
}
}

165
Code/DataFactory/dataflow/UpdateProductDimension.json Normal file
View File

@ -0,0 +1,165 @@
{
"name": "UpdateProductDimension",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}

169
Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json Normal file
View File

@ -0,0 +1,169 @@
{
"name": "UpdateProductDimension_Sorted",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
},
{
"name": "SortBySubcategory"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
"SortBySubcategory sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}

22
Code/DataFactory/dataset/ADLS_BIN_AWProduct.json Normal file
View File

@ -0,0 +1,22 @@
{
"name": "ADLS_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Product.csv",
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}

30
Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json Normal file
View File

@ -0,0 +1,30 @@
{
"name": "ADLS_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().Entity}.tsv",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}

24
Code/DataFactory/dataset/ADLS_DimProduct.json
View File

@ -1,24 +0,0 @@
{
"name": "ADLS_DimProduct",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab4"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"folderPath": "Cleaned/DimProduct",
"fileSystem": "lakeroot"
},
"compressionCodec": "snappy"
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

28
Code/DataFactory/dataset/ADLS_Product.json
View File

@ -1,28 +0,0 @@
{
"name": "ADLS_Product",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab4"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "SalesLT.Product.txt",
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"firstRowAsHeader": true,
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

28
Code/DataFactory/dataset/ADLS_ProductCategory.json
View File

@ -1,28 +0,0 @@
{
"name": "ADLS_ProductCategory",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab4"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "SalesLT.ProductCategory.txt",
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"firstRowAsHeader": true,
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

28
Code/DataFactory/dataset/ADLS_ProductModel.json
View File

@ -1,28 +0,0 @@
{
"name": "ADLS_ProductModel",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab4"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "SalesLT.ProductModel.txt",
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"firstRowAsHeader": true,
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

27
Code/DataFactory/dataset/ADLS_Raw.json
View File

@ -1,27 +0,0 @@
{
"name": "ADLS_Raw",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab2"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"firstRowAsHeader": true,
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

16
Code/DataFactory/dataset/ADLS_Raw_Generic.json → Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json
View File

@ -1,17 +1,17 @@
{
"name": "ADLS_Raw_Generic",
"name": "ADLS_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "AzureDataLakeStorage1",
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"File": {
"FileName": {
"type": "string"
}
},
"folder": {
"name": "Lab5"
"name": "Labs"
},
"annotations": [],
"type": "DelimitedText",
@ -19,18 +19,16 @@
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"value": "@dataset().FileName",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": ",",
"columnDelimiter": "\t",
"escapeChar": "\\",
"firstRowAsHeader": true,
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}
}

34
Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json
View File

@ -1,34 +0,0 @@
{
"name": "ASQL_AdventureWorks_Generic",
"properties": {
"linkedServiceName": {
"referenceName": "AzureSqlDatabase1",
"type": "LinkedServiceReference"
},
"parameters": {
"TableSchema": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "Lab5"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().TableSchema",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

106
Code/DataFactory/dataset/ASQL_Product.json
View File

@ -1,106 +0,0 @@
{
"name": "ASQL_Product",
"properties": {
"linkedServiceName": {
"referenceName": "AzureSqlDatabase1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab2"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "ProductID",
"type": "int",
"precision": 10
},
{
"name": "Name",
"type": "nvarchar"
},
{
"name": "ProductNumber",
"type": "nvarchar"
},
{
"name": "Color",
"type": "nvarchar"
},
{
"name": "StandardCost",
"type": "money",
"precision": 19,
"scale": 4
},
{
"name": "ListPrice",
"type": "money",
"precision": 19,
"scale": 4
},
{
"name": "Size",
"type": "nvarchar"
},
{
"name": "Weight",
"type": "decimal",
"precision": 8,
"scale": 2
},
{
"name": "ProductCategoryID",
"type": "int",
"precision": 10
},
{
"name": "ProductModelID",
"type": "int",
"precision": 10
},
{
"name": "SellStartDate",
"type": "datetime",
"precision": 23,
"scale": 3
},
{
"name": "SellEndDate",
"type": "datetime",
"precision": 23,
"scale": 3
},
{
"name": "DiscontinuedDate",
"type": "datetime",
"precision": 23,
"scale": 3
},
{
"name": "ThumbNailPhoto",
"type": "varbinary"
},
{
"name": "ThumbnailPhotoFileName",
"type": "nvarchar"
},
{
"name": "rowguid",
"type": "uniqueidentifier"
},
{
"name": "ModifiedDate",
"type": "datetime",
"precision": 23,
"scale": 3
}
],
"typeProperties": {
"schema": "SalesLT",
"table": "Product"
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

106
Code/DataFactory/dataset/ASQL_ProductModel.json
View File

@ -1,106 +0,0 @@
{
"name": "ASQL_ProductModel",
"properties": {
"linkedServiceName": {
"referenceName": "AzureSqlDatabase1",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lab4"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "ProductID",
"type": "int",
"precision": 10
},
{
"name": "Name",
"type": "nvarchar"
},
{
"name": "ProductNumber",
"type": "nvarchar"
},
{
"name": "Color",
"type": "nvarchar"
},
{
"name": "StandardCost",
"type": "money",
"precision": 19,
"scale": 4
},
{
"name": "ListPrice",
"type": "money",
"precision": 19,
"scale": 4
},
{
"name": "Size",
"type": "nvarchar"
},
{
"name": "Weight",
"type": "decimal",
"precision": 8,
"scale": 2
},
{
"name": "ProductCategoryID",
"type": "int",
"precision": 10
},
{
"name": "ProductModelID",
"type": "int",
"precision": 10
},
{
"name": "SellStartDate",
"type": "datetime",
"precision": 23,
"scale": 3
},
{
"name": "SellEndDate",
"type": "datetime",
"precision": 23,
"scale": 3
},
{
"name": "DiscontinuedDate",
"type": "datetime",
"precision": 23,
"scale": 3
},
{
"name": "ThumbNailPhoto",
"type": "varbinary"
},
{
"name": "ThumbnailPhotoFileName",
"type": "nvarchar"
},
{
"name": "rowguid",
"type": "uniqueidentifier"
},
{
"name": "ModifiedDate",
"type": "datetime",
"precision": 23,
"scale": 3
}
],
"typeProperties": {
"schema": "SalesLT",
"table": "ProductModel"
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

19
Code/DataFactory/dataset/HTTP_BIN_AWProduct.json Normal file
View File

@ -0,0 +1,19 @@
{
"name": "HTTP_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWProduct",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation"
}
}
}
}

28
Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json Normal file
View File

@ -0,0 +1,28 @@
{
"name": "HTTP_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().Entity}.csv",
"type": "Expression"
}
}
}
}
}

18
Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json Normal file
View File

@ -0,0 +1,18 @@
{
"name": "ADLS_saintegrationpipelines",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
}
}

12
Code/DataFactory/linkedService/HTTP_AWGitHub.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "HTTP_AWGitHub",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}

12
Code/DataFactory/linkedService/HTTP_AWProduct.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "HTTP_AWProduct",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}

46
Code/DataFactory/pipeline/BuildDimProduct.json Normal file
View File

@ -0,0 +1,46 @@
{
"name": "BuildDimProduct",
"properties": {
"activities": [
{
"name": "Run UpdateProductDimension",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "UpdateProductDimension",
"type": "DataFlowReference",
"datasetParameters": {
"Product": {
"FileName": "Product.tsv"
},
"ProductSubcategory": {
"FileName": "ProductSubcategory.tsv"
},
"ProductCategory": {
"FileName": "ProductCategory.tsv"
}
}
},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
}
],
"folder": {
"name": "Labs"
},
"annotations": []
}
}

36
Code/DataFactory/pipeline/BuildProductDimension.json
View File

@ -1,36 +0,0 @@
{
"name": "BuildProductDimension",
"properties": {
"activities": [
{
"name": "BuildProductDimension",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "BuildProductDimension",
"type": "DataFlowReference"
},
"compute": {
"coreCount": 8,
"computeType": "General"
}
}
}
],
"folder": {
"name": "Lab4"
},
"annotations": [],
"lastPublishTime": "2020-09-26T17:05:06Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

55
Code/DataFactory/pipeline/CopyAWEntities.json Normal file
View File

@ -0,0 +1,55 @@
{
"name": "CopyAWEntities",
"properties": {
"activities": [
{
"name": "Copy each AW entity",
"type": "ForEach",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@variables('Entities')",
"type": "Expression"
},
"isSequential": false,
"activities": [
{
"name": "Execute CopyAWEntity",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "CopyAWEntity",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"Entity": {
"value": "@item()",
"type": "Expression"
}
}
}
}
]
}
}
],
"variables": {
"Entities": {
"type": "Array",
"defaultValue": [
"Product",
"ProductSubcategory",
"ProductCategory"
]
}
},
"folder": {
"name": "Labs"
},
"annotations": []
}
}

72
Code/DataFactory/pipeline/CopyAWEntity.json Normal file
View File

@ -0,0 +1,72 @@
{
"name": "CopyAWEntity",
"properties": {
"activities": [
{
"name": "Copy AW entity",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AdventureWorks",
"type": "DatasetReference",
"parameters": {
"Entity": {
"value": "@pipeline().parameters.Entity",
"type": "Expression"
}
}
}
]
}
],
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs"
},
"annotations": []
}
}

55
Code/DataFactory/pipeline/CopyAWProduct.json Normal file
View File

@ -0,0 +1,55 @@
{
"name": "CopyAWProduct",
"properties": {
"activities": [
{
"name": "Copy AW Products to data lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "BinarySource",
"storeSettings": {
"type": "HttpReadSettings",
"requestMethod": "GET"
},
"formatSettings": {
"type": "BinaryReadSettings"
}
},
"sink": {
"type": "BinarySink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "HTTP_BIN_AWProduct",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_BIN_AWProduct",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Labs"
},
"annotations": []
}
}

83
Code/DataFactory/pipeline/CopyAdventureWorks.json
View File

@ -1,83 +0,0 @@
{
"name": "CopyAdventureWorks",
"properties": {
"activities": [
{
"name": "Get table list",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": "SELECT TABLE_SCHEMA, TABLE_NAME \nFROM INFORMATION_SCHEMA.TABLES\nWHERE TABLE_TYPE = 'BASE TABLE'\nAND TABLE_SCHEMA = 'SalesLT'\n",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "ASQL_Product",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "ForEach1",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get table list",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get table list').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Execute Pipeline1",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "CopyAnyTable",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"TableSchema": {
"value": "@item().TABLE_SCHEMA",
"type": "Expression"
},
"TableName": {
"value": "@item().TABLE_NAME",
"type": "Expression"
}
}
}
}
]
}
}
],
"folder": {
"name": "Lab5"
},
"annotations": [],
"lastPublishTime": "2020-09-26T17:11:52Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

89
Code/DataFactory/pipeline/CopyAnyTable.json
View File

@ -1,89 +0,0 @@
{
"name": "CopyAnyTable",
"properties": {
"activities": [
{
"name": "Copy table",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "ASQL_AdventureWorks_Generic",
"type": "DatasetReference",
"parameters": {
"TableSchema": {
"value": "@pipeline().parameters.TableSchema",
"type": "Expression"
},
"TableName": {
"value": "@pipeline().parameters.TableName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_Raw_Generic",
"type": "DatasetReference",
"parameters": {
"File": {
"value": "@{pipeline().parameters.TableSchema}.@{pipeline().parameters.TableName}.txt",
"type": "Expression"
}
}
}
]
}
],
"parameters": {
"TableSchema": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "Lab5"
},
"annotations": [],
"lastPublishTime": "2020-09-26T17:11:52Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

65
Code/DataFactory/pipeline/CopyProduct.json
View File

@ -1,65 +0,0 @@
{
"name": "CopyProduct",
"properties": {
"activities": [
{
"name": "Copy data to lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "ASQL_Product",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_Raw",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab2"
},
"annotations": [],
"lastPublishTime": "2020-09-26T14:33:50Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

38
Code/DataFactory/pipeline/CopyProductCategory.json
View File

@ -1,38 +0,0 @@
{
"name": "CopyProductCategory",
"properties": {
"activities": [
{
"name": "Execute CopyProductCategory",
"type": "ExecuteSSISPackage",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"packageLocation": {
"packagePath": "ADayFullOfAdf/SsisPackages/CopyProductCategory.dtsx",
"type": "SSISDB"
},
"environmentPath": null,
"connectVia": {
"referenceName": "SSISIntegrationRuntime",
"type": "IntegrationRuntimeReference"
},
"loggingLevel": "Basic"
}
}
],
"folder": {
"name": "Lab3"
},
"annotations": [],
"lastPublishTime": "2020-09-26T15:27:22Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

65
Code/DataFactory/pipeline/CopyProductModel.json
View File

@ -1,65 +0,0 @@
{
"name": "CopyProductModel",
"properties": {
"activities": [
{
"name": "Copy data1",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "ASQL_ProductModel",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_Raw",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab4"
},
"annotations": [],
"lastPublishTime": "2020-09-26T16:27:34Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

BIN
Labs/Lab 1 - Create Azure resources.pdf
View File

Binary file not shown.

BIN
Labs/Lab 2 - Build a Copy data pipeline.pdf
View File

Binary file not shown.

BIN
Labs/Lab 2 - Build a copy pipeline.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 3 - Create a reusable pipeline.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 3 - Run SSIS packages in ADF.pdf
View File

Binary file not shown.

BIN
Labs/Lab 4 - Author a data flow.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 4 - Build a Mapping Data Flow.pdf
View File

Binary file not shown.

BIN
Labs/Lab 5 - Create a reusable pipeline.pdf
View File

Binary file not shown.

BIN
Labs/Lab 5 - Monitor factory activity.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 6 - Explore Synapse pipelines.pdf Normal file
View File

Binary file not shown.

BIN
Labs/Lab 6 - Monitor factory activity.pdf
View File

Binary file not shown.