Merge pull request #1 from mrpaulandrewltd/lab-revisions

Lab revisions

Code/DataFactory/dataflow/BuildProductDimension.json

@@ -1,64 +0,0 @@
-{
-	"name": "BuildProductDimension",
-	"properties": {
-		"folder": {
-			"name": "Lab4"
-		},
-		"type": "MappingDataFlow",
-		"typeProperties": {
-			"sources": [
-				{
-					"dataset": {
-						"referenceName": "ADLS_Product",
-						"type": "DatasetReference"
-					},
-					"name": "Product"
-				},
-				{
-					"dataset": {
-						"referenceName": "ADLS_ProductCategory",
-						"type": "DatasetReference"
-					},
-					"name": "ProductCategory"
-				},
-				{
-					"dataset": {
-						"referenceName": "ADLS_ProductModel",
-						"type": "DatasetReference"
-					},
-					"name": "ProductModel"
-				}
-			],
-			"sinks": [
-				{
-					"dataset": {
-						"referenceName": "ADLS_DimProduct",
-						"type": "DatasetReference"
-					},
-					"name": "DimProduct"
-				}
-			],
-			"transformations": [
-				{
-					"name": "SelectProductColumns"
-				},
-				{
-					"name": "SelectProductCategoryColumns"
-				},
-				{
-					"name": "SelectProductModelColumns"
-				},
-				{
-					"name": "LookupProductCategory"
-				},
-				{
-					"name": "LookupProductModelID"
-				},
-				{
-					"name": "RemoveDuplicateColumns"
-				}
-			],
-			"script": "source(output(\n\t\tProductID as short,\n\t\tName as string,\n\t\tProductNumber as string,\n\t\tColor as string,\n\t\tStandardCost as double,\n\t\tListPrice as double,\n\t\tSize as string,\n\t\tWeight as double,\n\t\tProductCategoryID as short,\n\t\tProductModelID as short,\n\t\tSellStartDate as string,\n\t\tSellEndDate as string,\n\t\tDiscontinuedDate as string,\n\t\tThumbNailPhoto as string,\n\t\tThumbnailPhotoFileName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> Product\nsource(output(\n\t\tProductCategoryID as short,\n\t\tParentProductCategoryID as string,\n\t\tName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductCategory\nsource(output(\n\t\tProductModelID as short,\n\t\tName as string,\n\t\tCatalogDescription as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductModel\nProduct select(mapColumn(\n\t\tProductID,\n\t\tProduct = Name,\n\t\tProductCategoryID,\n\t\tProductModelID\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductColumns\nProductCategory select(mapColumn(\n\t\tProductCategoryID,\n\t\tProductCategory = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns\nProductModel select(mapColumn(\n\t\tProductModelID,\n\t\tProductModel = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductModelColumns\nSelectProductColumns, SelectProductCategoryColumns lookup(SelectProductColumns@ProductCategoryID == SelectProductCategoryColumns@ProductCategoryID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductCategory\nLookupProductCategory, SelectProductModelColumns lookup(SelectProductColumns@ProductModelID == SelectProductModelColumns@ProductModelID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductModelID\nLookupProductModelID select(mapColumn(\n\t\tProductID,\n\t\tProduct,\n\t\tProductCategoryID = {SelectProductColumns@ProductCategoryID},\n\t\tProductModelID = {SelectProductColumns@ProductModelID},\n\t\tProductCategory,\n\t\tProductModel\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns\nRemoveDuplicateColumns sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'parquet',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> DimProduct"
-		}
-	}
-}

Code/DataFactory/dataflow/UpdateProductDimension.json

@@ -0,0 +1,165 @@
+{
+	"name": "UpdateProductDimension",
+	"properties": {
+		"folder": {
+			"name": "Labs"
+		},
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "Product"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductSubcategory"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductCategory"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "ADLS_saintegrationpipelines",
+						"type": "LinkedServiceReference"
+					},
+					"name": "WriteToDataLake"
+				}
+			],
+			"transformations": [
+				{
+					"name": "SelectProductColumns"
+				},
+				{
+					"name": "SelectSubcategoryColumns"
+				},
+				{
+					"name": "SelectCategoryColumns"
+				},
+				{
+					"name": "LookupProductCategory"
+				},
+				{
+					"name": "LookupProductSubcategory"
+				},
+				{
+					"name": "RemoveDuplicateColumns"
+				}
+			],
+			"scriptLines": [
+				"source(output(",
+				"          ProductId as integer,",
+				"          Product as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as boolean,",
+				"          {_col4_} as boolean,",
+				"          {_col5_} as string,",
+				"          {_col6_} as short,",
+				"          {_col7_} as short,",
+				"          {_col8_} as double,",
+				"          {_col9_} as double,",
+				"          {_col10_} as string,",
+				"          {_col11_} as string,",
+				"          {_col12_} as string,",
+				"          {_col13_} as double,",
+				"          {_col14_} as short,",
+				"          {_col15_} as string,",
+				"          {_col16_} as string,",
+				"          {_col17_} as string,",
+				"          SubcategoryId as integer,",
+				"          {_col19_} as short,",
+				"          {_col20_} as timestamp,",
+				"          {_col21_} as timestamp,",
+				"          {_col22_} as string,",
+				"          {_col23_} as string,",
+				"          {_col24_} as string",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> Product",
+				"source(output(",
+				"          SubcategoryId as integer,",
+				"          CategoryId as integer,",
+				"          Subcategory as string,",
+				"          {_col3_} as string,",
+				"          {_col4_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductSubcategory",
+				"source(output(",
+				"          CategoryId as integer,",
+				"          Category as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductCategory",
+				"Product select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectProductColumns",
+				"ProductSubcategory select(mapColumn(",
+				"          SubcategoryId,",
+				"          CategoryId,",
+				"          Subcategory",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
+				"ProductCategory select(mapColumn(",
+				"          CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
+				"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductCategory",
+				"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductSubcategory",
+				"LookupProductSubcategory select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId = SelectProductColumns@SubcategoryId,",
+				"          SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
+				"          CategoryId = SelectSubcategoryColumns@CategoryId,",
+				"          Subcategory,",
+				"          CategoryId = SelectCategoryColumns@CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
+				"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'parquet',",
+				"     fileSystem: 'lakeroot',",
+				"     folderPath: 'Conformed/DimProduct',",
+				"     truncate: true,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> WriteToDataLake"
+			]
+		}
+	}
+}

Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json

@@ -0,0 +1,169 @@
+{
+	"name": "UpdateProductDimension_Sorted",
+	"properties": {
+		"folder": {
+			"name": "Labs"
+		},
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "Product"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductSubcategory"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductCategory"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "ADLS_saintegrationpipelines",
+						"type": "LinkedServiceReference"
+					},
+					"name": "WriteToDataLake"
+				}
+			],
+			"transformations": [
+				{
+					"name": "SelectProductColumns"
+				},
+				{
+					"name": "SelectSubcategoryColumns"
+				},
+				{
+					"name": "SelectCategoryColumns"
+				},
+				{
+					"name": "LookupProductCategory"
+				},
+				{
+					"name": "LookupProductSubcategory"
+				},
+				{
+					"name": "RemoveDuplicateColumns"
+				},
+				{
+					"name": "SortBySubcategory"
+				}
+			],
+			"scriptLines": [
+				"source(output(",
+				"          ProductId as integer,",
+				"          Product as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as boolean,",
+				"          {_col4_} as boolean,",
+				"          {_col5_} as string,",
+				"          {_col6_} as short,",
+				"          {_col7_} as short,",
+				"          {_col8_} as double,",
+				"          {_col9_} as double,",
+				"          {_col10_} as string,",
+				"          {_col11_} as string,",
+				"          {_col12_} as string,",
+				"          {_col13_} as double,",
+				"          {_col14_} as short,",
+				"          {_col15_} as string,",
+				"          {_col16_} as string,",
+				"          {_col17_} as string,",
+				"          SubcategoryId as integer,",
+				"          {_col19_} as short,",
+				"          {_col20_} as timestamp,",
+				"          {_col21_} as timestamp,",
+				"          {_col22_} as string,",
+				"          {_col23_} as string,",
+				"          {_col24_} as string",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> Product",
+				"source(output(",
+				"          SubcategoryId as integer,",
+				"          CategoryId as integer,",
+				"          Subcategory as string,",
+				"          {_col3_} as string,",
+				"          {_col4_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductSubcategory",
+				"source(output(",
+				"          CategoryId as integer,",
+				"          Category as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductCategory",
+				"Product select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectProductColumns",
+				"ProductSubcategory select(mapColumn(",
+				"          SubcategoryId,",
+				"          CategoryId,",
+				"          Subcategory",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
+				"ProductCategory select(mapColumn(",
+				"          CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
+				"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductCategory",
+				"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductSubcategory",
+				"LookupProductSubcategory select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId = SelectProductColumns@SubcategoryId,",
+				"          SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
+				"          CategoryId = SelectSubcategoryColumns@CategoryId,",
+				"          Subcategory,",
+				"          CategoryId = SelectCategoryColumns@CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
+				"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
+				"SortBySubcategory sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'parquet',",
+				"     fileSystem: 'lakeroot',",
+				"     folderPath: 'Conformed/DimProduct',",
+				"     truncate: true,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> WriteToDataLake"
+			]
+		}
+	}
+}

Code/DataFactory/dataset/ADLS_BIN_AWProduct.json

@@ -0,0 +1,22 @@
+{
+	"name": "ADLS_BIN_AWProduct",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "Product.csv",
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json

@@ -0,0 +1,30 @@
+{
+	"name": "ADLS_BIN_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@{dataset().Entity}.tsv",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/ADLS_DimProduct.json

@@ -1,24 +0,0 @@
-{
-	"name": "ADLS_DimProduct",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureDataLakeStorage1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"type": "Parquet",
-		"typeProperties": {
-			"location": {
-				"type": "AzureBlobFSLocation",
-				"folderPath": "Cleaned/DimProduct",
-				"fileSystem": "lakeroot"
-			},
-			"compressionCodec": "snappy"
-		},
-		"schema": []
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ADLS_Product.json

@@ -1,28 +0,0 @@
-{
-	"name": "ADLS_Product",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureDataLakeStorage1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"type": "DelimitedText",
-		"typeProperties": {
-			"location": {
-				"type": "AzureBlobFSLocation",
-				"fileName": "SalesLT.Product.txt",
-				"folderPath": "Raw",
-				"fileSystem": "lakeroot"
-			},
-			"columnDelimiter": ",",
-			"escapeChar": "\\",
-			"firstRowAsHeader": true,
-			"quoteChar": "\""
-		},
-		"schema": []
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ADLS_ProductCategory.json

@@ -1,28 +0,0 @@
-{
-	"name": "ADLS_ProductCategory",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureDataLakeStorage1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"type": "DelimitedText",
-		"typeProperties": {
-			"location": {
-				"type": "AzureBlobFSLocation",
-				"fileName": "SalesLT.ProductCategory.txt",
-				"folderPath": "Raw",
-				"fileSystem": "lakeroot"
-			},
-			"columnDelimiter": ",",
-			"escapeChar": "\\",
-			"firstRowAsHeader": true,
-			"quoteChar": "\""
-		},
-		"schema": []
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ADLS_ProductModel.json

@@ -1,28 +0,0 @@
-{
-	"name": "ADLS_ProductModel",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureDataLakeStorage1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"type": "DelimitedText",
-		"typeProperties": {
-			"location": {
-				"type": "AzureBlobFSLocation",
-				"fileName": "SalesLT.ProductModel.txt",
-				"folderPath": "Raw",
-				"fileSystem": "lakeroot"
-			},
-			"columnDelimiter": ",",
-			"escapeChar": "\\",
-			"firstRowAsHeader": true,
-			"quoteChar": "\""
-		},
-		"schema": []
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ADLS_Raw.json

@@ -1,27 +0,0 @@
-{
-	"name": "ADLS_Raw",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureDataLakeStorage1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab2"
-		},
-		"annotations": [],
-		"type": "DelimitedText",
-		"typeProperties": {
-			"location": {
-				"type": "AzureBlobFSLocation",
-				"folderPath": "Raw",
-				"fileSystem": "lakeroot"
-			},
-			"columnDelimiter": ",",
-			"escapeChar": "\\",
-			"firstRowAsHeader": true,
-			"quoteChar": "\""
-		},
-		"schema": []
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json

@@ -1,17 +1,17 @@
 {
-	"name": "ADLS_Raw_Generic",
+	"name": "ADLS_TSV_AdventureWorks",
 	"properties": {
 		"linkedServiceName": {
-			"referenceName": "AzureDataLakeStorage1",
+			"referenceName": "ADLS_saintegrationpipelines",
 			"type": "LinkedServiceReference"
 		},
 		"parameters": {
-			"File": {
+			"FileName": {
 				"type": "string"
 			}
 		},
 		"folder": {
-			"name": "Lab5"
+			"name": "Labs"
 		},
 		"annotations": [],
 		"type": "DelimitedText",
@@ -19,18 +19,16 @@
 			"location": {
 				"type": "AzureBlobFSLocation",
 				"fileName": {
-					"value": "@dataset().File",
+					"value": "@dataset().FileName",
 					"type": "Expression"
 				},
 				"folderPath": "Raw",
 				"fileSystem": "lakeroot"
 			},
-			"columnDelimiter": ",",
+			"columnDelimiter": "\t",
 			"escapeChar": "\\",
-			"firstRowAsHeader": true,
 			"quoteChar": "\""
 		},
 		"schema": []
-	},
+	}
-	"type": "Microsoft.DataFactory/factories/datasets"
 }

Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json

@@ -1,34 +0,0 @@
-{
-	"name": "ASQL_AdventureWorks_Generic",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureSqlDatabase1",
-			"type": "LinkedServiceReference"
-		},
-		"parameters": {
-			"TableSchema": {
-				"type": "string"
-			},
-			"TableName": {
-				"type": "string"
-			}
-		},
-		"folder": {
-			"name": "Lab5"
-		},
-		"annotations": [],
-		"type": "AzureSqlTable",
-		"schema": [],
-		"typeProperties": {
-			"schema": {
-				"value": "@dataset().TableSchema",
-				"type": "Expression"
-			},
-			"table": {
-				"value": "@dataset().TableName",
-				"type": "Expression"
-			}
-		}
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ASQL_Product.json

@@ -1,106 +0,0 @@
-{
-	"name": "ASQL_Product",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureSqlDatabase1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab2"
-		},
-		"annotations": [],
-		"type": "AzureSqlTable",
-		"schema": [
-			{
-				"name": "ProductID",
-				"type": "int",
-				"precision": 10
-			},
-			{
-				"name": "Name",
-				"type": "nvarchar"
-			},
-			{
-				"name": "ProductNumber",
-				"type": "nvarchar"
-			},
-			{
-				"name": "Color",
-				"type": "nvarchar"
-			},
-			{
-				"name": "StandardCost",
-				"type": "money",
-				"precision": 19,
-				"scale": 4
-			},
-			{
-				"name": "ListPrice",
-				"type": "money",
-				"precision": 19,
-				"scale": 4
-			},
-			{
-				"name": "Size",
-				"type": "nvarchar"
-			},
-			{
-				"name": "Weight",
-				"type": "decimal",
-				"precision": 8,
-				"scale": 2
-			},
-			{
-				"name": "ProductCategoryID",
-				"type": "int",
-				"precision": 10
-			},
-			{
-				"name": "ProductModelID",
-				"type": "int",
-				"precision": 10
-			},
-			{
-				"name": "SellStartDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			},
-			{
-				"name": "SellEndDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			},
-			{
-				"name": "DiscontinuedDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			},
-			{
-				"name": "ThumbNailPhoto",
-				"type": "varbinary"
-			},
-			{
-				"name": "ThumbnailPhotoFileName",
-				"type": "nvarchar"
-			},
-			{
-				"name": "rowguid",
-				"type": "uniqueidentifier"
-			},
-			{
-				"name": "ModifiedDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			}
-		],
-		"typeProperties": {
-			"schema": "SalesLT",
-			"table": "Product"
-		}
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/ASQL_ProductModel.json

@@ -1,106 +0,0 @@
-{
-	"name": "ASQL_ProductModel",
-	"properties": {
-		"linkedServiceName": {
-			"referenceName": "AzureSqlDatabase1",
-			"type": "LinkedServiceReference"
-		},
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"type": "AzureSqlTable",
-		"schema": [
-			{
-				"name": "ProductID",
-				"type": "int",
-				"precision": 10
-			},
-			{
-				"name": "Name",
-				"type": "nvarchar"
-			},
-			{
-				"name": "ProductNumber",
-				"type": "nvarchar"
-			},
-			{
-				"name": "Color",
-				"type": "nvarchar"
-			},
-			{
-				"name": "StandardCost",
-				"type": "money",
-				"precision": 19,
-				"scale": 4
-			},
-			{
-				"name": "ListPrice",
-				"type": "money",
-				"precision": 19,
-				"scale": 4
-			},
-			{
-				"name": "Size",
-				"type": "nvarchar"
-			},
-			{
-				"name": "Weight",
-				"type": "decimal",
-				"precision": 8,
-				"scale": 2
-			},
-			{
-				"name": "ProductCategoryID",
-				"type": "int",
-				"precision": 10
-			},
-			{
-				"name": "ProductModelID",
-				"type": "int",
-				"precision": 10
-			},
-			{
-				"name": "SellStartDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			},
-			{
-				"name": "SellEndDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			},
-			{
-				"name": "DiscontinuedDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			},
-			{
-				"name": "ThumbNailPhoto",
-				"type": "varbinary"
-			},
-			{
-				"name": "ThumbnailPhotoFileName",
-				"type": "nvarchar"
-			},
-			{
-				"name": "rowguid",
-				"type": "uniqueidentifier"
-			},
-			{
-				"name": "ModifiedDate",
-				"type": "datetime",
-				"precision": 23,
-				"scale": 3
-			}
-		],
-		"typeProperties": {
-			"schema": "SalesLT",
-			"table": "ProductModel"
-		}
-	},
-	"type": "Microsoft.DataFactory/factories/datasets"
-}

Code/DataFactory/dataset/HTTP_BIN_AWProduct.json

@@ -0,0 +1,19 @@
+{
+	"name": "HTTP_BIN_AWProduct",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWProduct",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json

@@ -0,0 +1,28 @@
+{
+	"name": "HTTP_BIN_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWGitHub",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation",
+				"relativeUrl": {
+					"value": "@{dataset().Entity}.csv",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json

@@ -0,0 +1,18 @@
+{
+	"name": "ADLS_saintegrationpipelines",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"accountKey": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "traininglake01"
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/HTTP_AWGitHub.json

@@ -0,0 +1,12 @@
+{
+	"name": "HTTP_AWGitHub",
+	"properties": {
+		"annotations": [],
+		"type": "HttpServer",
+		"typeProperties": {
+			"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
+			"enableServerCertificateValidation": true,
+			"authenticationType": "Anonymous"
+		}
+	}
+}

Code/DataFactory/linkedService/HTTP_AWProduct.json

@@ -0,0 +1,12 @@
+{
+	"name": "HTTP_AWProduct",
+	"properties": {
+		"annotations": [],
+		"type": "HttpServer",
+		"typeProperties": {
+			"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
+			"enableServerCertificateValidation": true,
+			"authenticationType": "Anonymous"
+		}
+	}
+}

Code/DataFactory/pipeline/BuildDimProduct.json

@@ -0,0 +1,46 @@
+{
+	"name": "BuildDimProduct",
+	"properties": {
+		"activities": [
+			{
+				"name": "Run UpdateProductDimension",
+				"type": "ExecuteDataFlow",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"dataflow": {
+						"referenceName": "UpdateProductDimension",
+						"type": "DataFlowReference",
+						"datasetParameters": {
+							"Product": {
+								"FileName": "Product.tsv"
+							},
+							"ProductSubcategory": {
+								"FileName": "ProductSubcategory.tsv"
+							},
+							"ProductCategory": {
+								"FileName": "ProductCategory.tsv"
+							}
+						}
+					},
+					"compute": {
+						"coreCount": 8,
+						"computeType": "General"
+					},
+					"traceLevel": "Fine"
+				}
+			}
+		],
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": []
+	}
+}

Code/DataFactory/pipeline/BuildProductDimension.json

@@ -1,36 +0,0 @@
-{
-	"name": "BuildProductDimension",
-	"properties": {
-		"activities": [
-			{
-				"name": "BuildProductDimension",
-				"type": "ExecuteDataFlow",
-				"dependsOn": [],
-				"policy": {
-					"timeout": "7.00:00:00",
-					"retry": 0,
-					"retryIntervalInSeconds": 30,
-					"secureOutput": false,
-					"secureInput": false
-				},
-				"userProperties": [],
-				"typeProperties": {
-					"dataflow": {
-						"referenceName": "BuildProductDimension",
-						"type": "DataFlowReference"
-					},
-					"compute": {
-						"coreCount": 8,
-						"computeType": "General"
-					}
-				}
-			}
-		],
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"lastPublishTime": "2020-09-26T17:05:06Z"
-	},
-	"type": "Microsoft.DataFactory/factories/pipelines"
-}

Code/DataFactory/pipeline/CopyAWEntities.json

@@ -0,0 +1,55 @@
+{
+	"name": "CopyAWEntities",
+	"properties": {
+		"activities": [
+			{
+				"name": "Copy each AW entity",
+				"type": "ForEach",
+				"dependsOn": [],
+				"userProperties": [],
+				"typeProperties": {
+					"items": {
+						"value": "@variables('Entities')",
+						"type": "Expression"
+					},
+					"isSequential": false,
+					"activities": [
+						{
+							"name": "Execute CopyAWEntity",
+							"type": "ExecutePipeline",
+							"dependsOn": [],
+							"userProperties": [],
+							"typeProperties": {
+								"pipeline": {
+									"referenceName": "CopyAWEntity",
+									"type": "PipelineReference"
+								},
+								"waitOnCompletion": true,
+								"parameters": {
+									"Entity": {
+										"value": "@item()",
+										"type": "Expression"
+									}
+								}
+							}
+						}
+					]
+				}
+			}
+		],
+		"variables": {
+			"Entities": {
+				"type": "Array",
+				"defaultValue": [
+					"Product",
+					"ProductSubcategory",
+					"ProductCategory"
+				]
+			}
+		},
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": []
+	}
+}

Code/DataFactory/pipeline/CopyAWEntity.json

@@ -0,0 +1,72 @@
+{
+	"name": "CopyAWEntity",
+	"properties": {
+		"activities": [
+			{
+				"name": "Copy AW entity",
+				"type": "Copy",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"source": {
+						"type": "BinarySource",
+						"storeSettings": {
+							"type": "HttpReadSettings",
+							"requestMethod": "GET"
+						},
+						"formatSettings": {
+							"type": "BinaryReadSettings"
+						}
+					},
+					"sink": {
+						"type": "BinarySink",
+						"storeSettings": {
+							"type": "AzureBlobFSWriteSettings"
+						}
+					},
+					"enableStaging": false
+				},
+				"inputs": [
+					{
+						"referenceName": "HTTP_BIN_AdventureWorks",
+						"type": "DatasetReference",
+						"parameters": {
+							"Entity": {
+								"value": "@pipeline().parameters.Entity",
+								"type": "Expression"
+							}
+						}
+					}
+				],
+				"outputs": [
+					{
+						"referenceName": "ADLS_BIN_AdventureWorks",
+						"type": "DatasetReference",
+						"parameters": {
+							"Entity": {
+								"value": "@pipeline().parameters.Entity",
+								"type": "Expression"
+							}
+						}
+					}
+				]
+			}
+		],
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": []
+	}
+}

Code/DataFactory/pipeline/CopyAWProduct.json

@@ -0,0 +1,55 @@
+{
+	"name": "CopyAWProduct",
+	"properties": {
+		"activities": [
+			{
+				"name": "Copy AW Products to data lake",
+				"type": "Copy",
+				"dependsOn": [],
+				"policy": {
+					"timeout": "0.12:00:00",
+					"retry": 0,
+					"retryIntervalInSeconds": 30,
+					"secureOutput": false,
+					"secureInput": false
+				},
+				"userProperties": [],
+				"typeProperties": {
+					"source": {
+						"type": "BinarySource",
+						"storeSettings": {
+							"type": "HttpReadSettings",
+							"requestMethod": "GET"
+						},
+						"formatSettings": {
+							"type": "BinaryReadSettings"
+						}
+					},
+					"sink": {
+						"type": "BinarySink",
+						"storeSettings": {
+							"type": "AzureBlobFSWriteSettings"
+						}
+					},
+					"enableStaging": false
+				},
+				"inputs": [
+					{
+						"referenceName": "HTTP_BIN_AWProduct",
+						"type": "DatasetReference"
+					}
+				],
+				"outputs": [
+					{
+						"referenceName": "ADLS_BIN_AWProduct",
+						"type": "DatasetReference"
+					}
+				]
+			}
+		],
+		"folder": {
+			"name": "Labs"
+		},
+		"annotations": []
+	}
+}

Code/DataFactory/pipeline/CopyAdventureWorks.json

@@ -1,83 +0,0 @@
-{
-	"name": "CopyAdventureWorks",
-	"properties": {
-		"activities": [
-			{
-				"name": "Get table list",
-				"type": "Lookup",
-				"dependsOn": [],
-				"policy": {
-					"timeout": "7.00:00:00",
-					"retry": 0,
-					"retryIntervalInSeconds": 30,
-					"secureOutput": false,
-					"secureInput": false
-				},
-				"userProperties": [],
-				"typeProperties": {
-					"source": {
-						"type": "AzureSqlSource",
-						"sqlReaderQuery": "SELECT TABLE_SCHEMA, TABLE_NAME \nFROM INFORMATION_SCHEMA.TABLES\nWHERE TABLE_TYPE = 'BASE TABLE'\nAND TABLE_SCHEMA = 'SalesLT'\n",
-						"queryTimeout": "02:00:00",
-						"partitionOption": "None"
-					},
-					"dataset": {
-						"referenceName": "ASQL_Product",
-						"type": "DatasetReference"
-					},
-					"firstRowOnly": false
-				}
-			},
-			{
-				"name": "ForEach1",
-				"type": "ForEach",
-				"dependsOn": [
-					{
-						"activity": "Get table list",
-						"dependencyConditions": [
-							"Succeeded"
-						]
-					}
-				],
-				"userProperties": [],
-				"typeProperties": {
-					"items": {
-						"value": "@activity('Get table list').output.value",
-						"type": "Expression"
-					},
-					"activities": [
-						{
-							"name": "Execute Pipeline1",
-							"type": "ExecutePipeline",
-							"dependsOn": [],
-							"userProperties": [],
-							"typeProperties": {
-								"pipeline": {
-									"referenceName": "CopyAnyTable",
-									"type": "PipelineReference"
-								},
-								"waitOnCompletion": true,
-								"parameters": {
-									"TableSchema": {
-										"value": "@item().TABLE_SCHEMA",
-										"type": "Expression"
-									},
-									"TableName": {
-										"value": "@item().TABLE_NAME",
-										"type": "Expression"
-									}
-								}
-							}
-						}
-					]
-				}
-			}
-		],
-		"folder": {
-			"name": "Lab5"
-		},
-		"annotations": [],
-		"lastPublishTime": "2020-09-26T17:11:52Z"
-	},
-	"type": "Microsoft.DataFactory/factories/pipelines"
-}

Code/DataFactory/pipeline/CopyAnyTable.json

@@ -1,89 +0,0 @@
-{
-	"name": "CopyAnyTable",
-	"properties": {
-		"activities": [
-			{
-				"name": "Copy table",
-				"type": "Copy",
-				"dependsOn": [],
-				"policy": {
-					"timeout": "7.00:00:00",
-					"retry": 0,
-					"retryIntervalInSeconds": 30,
-					"secureOutput": false,
-					"secureInput": false
-				},
-				"userProperties": [],
-				"typeProperties": {
-					"source": {
-						"type": "AzureSqlSource",
-						"queryTimeout": "02:00:00",
-						"partitionOption": "None"
-					},
-					"sink": {
-						"type": "DelimitedTextSink",
-						"storeSettings": {
-							"type": "AzureBlobFSWriteSettings"
-						},
-						"formatSettings": {
-							"type": "DelimitedTextWriteSettings",
-							"quoteAllText": true,
-							"fileExtension": ".txt"
-						}
-					},
-					"enableStaging": false,
-					"translator": {
-						"type": "TabularTranslator",
-						"typeConversion": true,
-						"typeConversionSettings": {
-							"allowDataTruncation": true,
-							"treatBooleanAsNumber": false
-						}
-					}
-				},
-				"inputs": [
-					{
-						"referenceName": "ASQL_AdventureWorks_Generic",
-						"type": "DatasetReference",
-						"parameters": {
-							"TableSchema": {
-								"value": "@pipeline().parameters.TableSchema",
-								"type": "Expression"
-							},
-							"TableName": {
-								"value": "@pipeline().parameters.TableName",
-								"type": "Expression"
-							}
-						}
-					}
-				],
-				"outputs": [
-					{
-						"referenceName": "ADLS_Raw_Generic",
-						"type": "DatasetReference",
-						"parameters": {
-							"File": {
-								"value": "@{pipeline().parameters.TableSchema}.@{pipeline().parameters.TableName}.txt",
-								"type": "Expression"
-							}
-						}
-					}
-				]
-			}
-		],
-		"parameters": {
-			"TableSchema": {
-				"type": "string"
-			},
-			"TableName": {
-				"type": "string"
-			}
-		},
-		"folder": {
-			"name": "Lab5"
-		},
-		"annotations": [],
-		"lastPublishTime": "2020-09-26T17:11:52Z"
-	},
-	"type": "Microsoft.DataFactory/factories/pipelines"
-}

Code/DataFactory/pipeline/CopyProduct.json

@@ -1,65 +0,0 @@
-{
-	"name": "CopyProduct",
-	"properties": {
-		"activities": [
-			{
-				"name": "Copy data to lake",
-				"type": "Copy",
-				"dependsOn": [],
-				"policy": {
-					"timeout": "7.00:00:00",
-					"retry": 0,
-					"retryIntervalInSeconds": 30,
-					"secureOutput": false,
-					"secureInput": false
-				},
-				"userProperties": [],
-				"typeProperties": {
-					"source": {
-						"type": "AzureSqlSource",
-						"queryTimeout": "02:00:00",
-						"partitionOption": "None"
-					},
-					"sink": {
-						"type": "DelimitedTextSink",
-						"storeSettings": {
-							"type": "AzureBlobFSWriteSettings"
-						},
-						"formatSettings": {
-							"type": "DelimitedTextWriteSettings",
-							"quoteAllText": true,
-							"fileExtension": ".txt"
-						}
-					},
-					"enableStaging": false,
-					"translator": {
-						"type": "TabularTranslator",
-						"typeConversion": true,
-						"typeConversionSettings": {
-							"allowDataTruncation": true,
-							"treatBooleanAsNumber": false
-						}
-					}
-				},
-				"inputs": [
-					{
-						"referenceName": "ASQL_Product",
-						"type": "DatasetReference"
-					}
-				],
-				"outputs": [
-					{
-						"referenceName": "ADLS_Raw",
-						"type": "DatasetReference"
-					}
-				]
-			}
-		],
-		"folder": {
-			"name": "Lab2"
-		},
-		"annotations": [],
-		"lastPublishTime": "2020-09-26T14:33:50Z"
-	},
-	"type": "Microsoft.DataFactory/factories/pipelines"
-}

Code/DataFactory/pipeline/CopyProductCategory.json

@@ -1,38 +0,0 @@
-{
-	"name": "CopyProductCategory",
-	"properties": {
-		"activities": [
-			{
-				"name": "Execute CopyProductCategory",
-				"type": "ExecuteSSISPackage",
-				"dependsOn": [],
-				"policy": {
-					"timeout": "7.00:00:00",
-					"retry": 0,
-					"retryIntervalInSeconds": 30,
-					"secureOutput": false,
-					"secureInput": false
-				},
-				"userProperties": [],
-				"typeProperties": {
-					"packageLocation": {
-						"packagePath": "ADayFullOfAdf/SsisPackages/CopyProductCategory.dtsx",
-						"type": "SSISDB"
-					},
-					"environmentPath": null,
-					"connectVia": {
-						"referenceName": "SSISIntegrationRuntime",
-						"type": "IntegrationRuntimeReference"
-					},
-					"loggingLevel": "Basic"
-				}
-			}
-		],
-		"folder": {
-			"name": "Lab3"
-		},
-		"annotations": [],
-		"lastPublishTime": "2020-09-26T15:27:22Z"
-	},
-	"type": "Microsoft.DataFactory/factories/pipelines"
-}

Code/DataFactory/pipeline/CopyProductModel.json

@@ -1,65 +0,0 @@
-{
-	"name": "CopyProductModel",
-	"properties": {
-		"activities": [
-			{
-				"name": "Copy data1",
-				"type": "Copy",
-				"dependsOn": [],
-				"policy": {
-					"timeout": "7.00:00:00",
-					"retry": 0,
-					"retryIntervalInSeconds": 30,
-					"secureOutput": false,
-					"secureInput": false
-				},
-				"userProperties": [],
-				"typeProperties": {
-					"source": {
-						"type": "AzureSqlSource",
-						"queryTimeout": "02:00:00",
-						"partitionOption": "None"
-					},
-					"sink": {
-						"type": "DelimitedTextSink",
-						"storeSettings": {
-							"type": "AzureBlobFSWriteSettings"
-						},
-						"formatSettings": {
-							"type": "DelimitedTextWriteSettings",
-							"quoteAllText": true,
-							"fileExtension": ".txt"
-						}
-					},
-					"enableStaging": false,
-					"translator": {
-						"type": "TabularTranslator",
-						"typeConversion": true,
-						"typeConversionSettings": {
-							"allowDataTruncation": true,
-							"treatBooleanAsNumber": false
-						}
-					}
-				},
-				"inputs": [
-					{
-						"referenceName": "ASQL_ProductModel",
-						"type": "DatasetReference"
-					}
-				],
-				"outputs": [
-					{
-						"referenceName": "ADLS_Raw",
-						"type": "DatasetReference"
-					}
-				]
-			}
-		],
-		"folder": {
-			"name": "Lab4"
-		},
-		"annotations": [],
-		"lastPublishTime": "2020-09-26T16:27:34Z"
-	},
-	"type": "Microsoft.DataFactory/factories/pipelines"
-}