diff --git a/Code/DataFactory/dataflow/BuildProductDimension.json b/Code/DataFactory/dataflow/BuildProductDimension.json deleted file mode 100644 index 503aa5f..0000000 --- a/Code/DataFactory/dataflow/BuildProductDimension.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "BuildProductDimension", - "properties": { - "folder": { - "name": "Lab4" - }, - "type": "MappingDataFlow", - "typeProperties": { - "sources": [ - { - "dataset": { - "referenceName": "ADLS_Product", - "type": "DatasetReference" - }, - "name": "Product" - }, - { - "dataset": { - "referenceName": "ADLS_ProductCategory", - "type": "DatasetReference" - }, - "name": "ProductCategory" - }, - { - "dataset": { - "referenceName": "ADLS_ProductModel", - "type": "DatasetReference" - }, - "name": "ProductModel" - } - ], - "sinks": [ - { - "dataset": { - "referenceName": "ADLS_DimProduct", - "type": "DatasetReference" - }, - "name": "DimProduct" - } - ], - "transformations": [ - { - "name": "SelectProductColumns" - }, - { - "name": "SelectProductCategoryColumns" - }, - { - "name": "SelectProductModelColumns" - }, - { - "name": "LookupProductCategory" - }, - { - "name": "LookupProductModelID" - }, - { - "name": "RemoveDuplicateColumns" - } - ], - "script": "source(output(\n\t\tProductID as short,\n\t\tName as string,\n\t\tProductNumber as string,\n\t\tColor as string,\n\t\tStandardCost as double,\n\t\tListPrice as double,\n\t\tSize as string,\n\t\tWeight as double,\n\t\tProductCategoryID as short,\n\t\tProductModelID as short,\n\t\tSellStartDate as string,\n\t\tSellEndDate as string,\n\t\tDiscontinuedDate as string,\n\t\tThumbNailPhoto as string,\n\t\tThumbnailPhotoFileName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> Product\nsource(output(\n\t\tProductCategoryID as short,\n\t\tParentProductCategoryID as string,\n\t\tName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductCategory\nsource(output(\n\t\tProductModelID as short,\n\t\tName as string,\n\t\tCatalogDescription as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductModel\nProduct select(mapColumn(\n\t\tProductID,\n\t\tProduct = Name,\n\t\tProductCategoryID,\n\t\tProductModelID\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductColumns\nProductCategory select(mapColumn(\n\t\tProductCategoryID,\n\t\tProductCategory = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns\nProductModel select(mapColumn(\n\t\tProductModelID,\n\t\tProductModel = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductModelColumns\nSelectProductColumns, SelectProductCategoryColumns lookup(SelectProductColumns@ProductCategoryID == SelectProductCategoryColumns@ProductCategoryID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductCategory\nLookupProductCategory, SelectProductModelColumns lookup(SelectProductColumns@ProductModelID == SelectProductModelColumns@ProductModelID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductModelID\nLookupProductModelID select(mapColumn(\n\t\tProductID,\n\t\tProduct,\n\t\tProductCategoryID = {SelectProductColumns@ProductCategoryID},\n\t\tProductModelID = {SelectProductColumns@ProductModelID},\n\t\tProductCategory,\n\t\tProductModel\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns\nRemoveDuplicateColumns sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'parquet',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> DimProduct" - } - } -} \ No newline at end of file diff --git a/Code/DataFactory/dataflow/UpdateProductDimension.json b/Code/DataFactory/dataflow/UpdateProductDimension.json new file mode 100644 index 0000000..581a3be --- /dev/null +++ b/Code/DataFactory/dataflow/UpdateProductDimension.json @@ -0,0 +1,165 @@ +{ + "name": "UpdateProductDimension", + "properties": { + "folder": { + "name": "Labs" + }, + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "Product" + }, + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "ProductSubcategory" + }, + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "ProductCategory" + } + ], + "sinks": [ + { + "linkedService": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "name": "WriteToDataLake" + } + ], + "transformations": [ + { + "name": "SelectProductColumns" + }, + { + "name": "SelectSubcategoryColumns" + }, + { + "name": "SelectCategoryColumns" + }, + { + "name": "LookupProductCategory" + }, + { + "name": "LookupProductSubcategory" + }, + { + "name": "RemoveDuplicateColumns" + } + ], + "scriptLines": [ + "source(output(", + " ProductId as integer,", + " Product as string,", + " {_col2_} as string,", + " {_col3_} as boolean,", + " {_col4_} as boolean,", + " {_col5_} as string,", + " {_col6_} as short,", + " {_col7_} as short,", + " {_col8_} as double,", + " {_col9_} as double,", + " {_col10_} as string,", + " {_col11_} as string,", + " {_col12_} as string,", + " {_col13_} as double,", + " {_col14_} as short,", + " {_col15_} as string,", + " {_col16_} as string,", + " {_col17_} as string,", + " SubcategoryId as integer,", + " {_col19_} as short,", + " {_col20_} as timestamp,", + " {_col21_} as timestamp,", + " {_col22_} as string,", + " {_col23_} as string,", + " {_col24_} as string", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> Product", + "source(output(", + " SubcategoryId as integer,", + " CategoryId as integer,", + " Subcategory as string,", + " {_col3_} as string,", + " {_col4_} as timestamp", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> ProductSubcategory", + "source(output(", + " CategoryId as integer,", + " Category as string,", + " {_col2_} as string,", + " {_col3_} as timestamp", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> ProductCategory", + "Product select(mapColumn(", + " ProductId,", + " Product,", + " SubcategoryId", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectProductColumns", + "ProductSubcategory select(mapColumn(", + " SubcategoryId,", + " CategoryId,", + " Subcategory", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns", + "ProductCategory select(mapColumn(", + " CategoryId,", + " Category", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectCategoryColumns", + "SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,", + " multiple: false,", + " pickup: 'any',", + " broadcast: 'auto')~> LookupProductCategory", + "SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,", + " multiple: false,", + " pickup: 'any',", + " broadcast: 'auto')~> LookupProductSubcategory", + "LookupProductSubcategory select(mapColumn(", + " ProductId,", + " Product,", + " SubcategoryId = SelectProductColumns@SubcategoryId,", + " SubcategoryId = SelectSubcategoryColumns@SubcategoryId,", + " CategoryId = SelectSubcategoryColumns@CategoryId,", + " Subcategory,", + " CategoryId = SelectCategoryColumns@CategoryId,", + " Category", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns", + "RemoveDuplicateColumns sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'parquet',", + " fileSystem: 'lakeroot',", + " folderPath: 'Conformed/DimProduct',", + " truncate: true,", + " umask: 0022,", + " preCommands: [],", + " postCommands: [],", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> WriteToDataLake" + ] + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json b/Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json new file mode 100644 index 0000000..770765e --- /dev/null +++ b/Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json @@ -0,0 +1,169 @@ +{ + "name": "UpdateProductDimension_Sorted", + "properties": { + "folder": { + "name": "Labs" + }, + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "Product" + }, + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "ProductSubcategory" + }, + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "ProductCategory" + } + ], + "sinks": [ + { + "linkedService": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "name": "WriteToDataLake" + } + ], + "transformations": [ + { + "name": "SelectProductColumns" + }, + { + "name": "SelectSubcategoryColumns" + }, + { + "name": "SelectCategoryColumns" + }, + { + "name": "LookupProductCategory" + }, + { + "name": "LookupProductSubcategory" + }, + { + "name": "RemoveDuplicateColumns" + }, + { + "name": "SortBySubcategory" + } + ], + "scriptLines": [ + "source(output(", + " ProductId as integer,", + " Product as string,", + " {_col2_} as string,", + " {_col3_} as boolean,", + " {_col4_} as boolean,", + " {_col5_} as string,", + " {_col6_} as short,", + " {_col7_} as short,", + " {_col8_} as double,", + " {_col9_} as double,", + " {_col10_} as string,", + " {_col11_} as string,", + " {_col12_} as string,", + " {_col13_} as double,", + " {_col14_} as short,", + " {_col15_} as string,", + " {_col16_} as string,", + " {_col17_} as string,", + " SubcategoryId as integer,", + " {_col19_} as short,", + " {_col20_} as timestamp,", + " {_col21_} as timestamp,", + " {_col22_} as string,", + " {_col23_} as string,", + " {_col24_} as string", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> Product", + "source(output(", + " SubcategoryId as integer,", + " CategoryId as integer,", + " Subcategory as string,", + " {_col3_} as string,", + " {_col4_} as timestamp", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> ProductSubcategory", + "source(output(", + " CategoryId as integer,", + " Category as string,", + " {_col2_} as string,", + " {_col3_} as timestamp", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> ProductCategory", + "Product select(mapColumn(", + " ProductId,", + " Product,", + " SubcategoryId", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectProductColumns", + "ProductSubcategory select(mapColumn(", + " SubcategoryId,", + " CategoryId,", + " Subcategory", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns", + "ProductCategory select(mapColumn(", + " CategoryId,", + " Category", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectCategoryColumns", + "SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,", + " multiple: false,", + " pickup: 'any',", + " broadcast: 'auto')~> LookupProductCategory", + "SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,", + " multiple: false,", + " pickup: 'any',", + " broadcast: 'auto')~> LookupProductSubcategory", + "LookupProductSubcategory select(mapColumn(", + " ProductId,", + " Product,", + " SubcategoryId = SelectProductColumns@SubcategoryId,", + " SubcategoryId = SelectSubcategoryColumns@SubcategoryId,", + " CategoryId = SelectSubcategoryColumns@CategoryId,", + " Subcategory,", + " CategoryId = SelectCategoryColumns@CategoryId,", + " Category", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns", + "RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory", + "SortBySubcategory sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'parquet',", + " fileSystem: 'lakeroot',", + " folderPath: 'Conformed/DimProduct',", + " truncate: true,", + " umask: 0022,", + " preCommands: [],", + " postCommands: [],", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> WriteToDataLake" + ] + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json b/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json new file mode 100644 index 0000000..13f0ffd --- /dev/null +++ b/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json @@ -0,0 +1,22 @@ +{ + "name": "ADLS_BIN_AWProduct", + "properties": { + "linkedServiceName": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Labs" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "Product.csv", + "folderPath": "Raw", + "fileSystem": "lakeroot" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json b/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json new file mode 100644 index 0000000..934edf6 --- /dev/null +++ b/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json @@ -0,0 +1,30 @@ +{ + "name": "ADLS_BIN_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Labs" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@{dataset().Entity}.tsv", + "type": "Expression" + }, + "folderPath": "Raw", + "fileSystem": "lakeroot" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_DimProduct.json b/Code/DataFactory/dataset/ADLS_DimProduct.json deleted file mode 100644 index dcbdb00..0000000 --- a/Code/DataFactory/dataset/ADLS_DimProduct.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "ADLS_DimProduct", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "Parquet", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "folderPath": "Cleaned/DimProduct", - "fileSystem": "lakeroot" - }, - "compressionCodec": "snappy" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_Product.json b/Code/DataFactory/dataset/ADLS_Product.json deleted file mode 100644 index ab9a7b3..0000000 --- a/Code/DataFactory/dataset/ADLS_Product.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ADLS_Product", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": "SalesLT.Product.txt", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_ProductCategory.json b/Code/DataFactory/dataset/ADLS_ProductCategory.json deleted file mode 100644 index e357684..0000000 --- a/Code/DataFactory/dataset/ADLS_ProductCategory.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ADLS_ProductCategory", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": "SalesLT.ProductCategory.txt", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_ProductModel.json b/Code/DataFactory/dataset/ADLS_ProductModel.json deleted file mode 100644 index 013aea5..0000000 --- a/Code/DataFactory/dataset/ADLS_ProductModel.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ADLS_ProductModel", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": "SalesLT.ProductModel.txt", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_Raw.json b/Code/DataFactory/dataset/ADLS_Raw.json deleted file mode 100644 index 251c768..0000000 --- a/Code/DataFactory/dataset/ADLS_Raw.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "ADLS_Raw", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab2" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_Raw_Generic.json b/Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json similarity index 64% rename from Code/DataFactory/dataset/ADLS_Raw_Generic.json rename to Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json index e9d2448..c23301b 100644 --- a/Code/DataFactory/dataset/ADLS_Raw_Generic.json +++ b/Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json @@ -1,17 +1,17 @@ { - "name": "ADLS_Raw_Generic", + "name": "ADLS_TSV_AdventureWorks", "properties": { "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", + "referenceName": "ADLS_saintegrationpipelines", "type": "LinkedServiceReference" }, "parameters": { - "File": { + "FileName": { "type": "string" } }, "folder": { - "name": "Lab5" + "name": "Labs" }, "annotations": [], "type": "DelimitedText", @@ -19,18 +19,16 @@ "location": { "type": "AzureBlobFSLocation", "fileName": { - "value": "@dataset().File", + "value": "@dataset().FileName", "type": "Expression" }, "folderPath": "Raw", "fileSystem": "lakeroot" }, - "columnDelimiter": ",", + "columnDelimiter": "\t", "escapeChar": "\\", - "firstRowAsHeader": true, "quoteChar": "\"" }, "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" + } } \ No newline at end of file diff --git a/Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json b/Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json deleted file mode 100644 index 8339d81..0000000 --- a/Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "ASQL_AdventureWorks_Generic", - "properties": { - "linkedServiceName": { - "referenceName": "AzureSqlDatabase1", - "type": "LinkedServiceReference" - }, - "parameters": { - "TableSchema": { - "type": "string" - }, - "TableName": { - "type": "string" - } - }, - "folder": { - "name": "Lab5" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [], - "typeProperties": { - "schema": { - "value": "@dataset().TableSchema", - "type": "Expression" - }, - "table": { - "value": "@dataset().TableName", - "type": "Expression" - } - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ASQL_Product.json b/Code/DataFactory/dataset/ASQL_Product.json deleted file mode 100644 index ebe1cc1..0000000 --- a/Code/DataFactory/dataset/ASQL_Product.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "name": "ASQL_Product", - "properties": { - "linkedServiceName": { - "referenceName": "AzureSqlDatabase1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab2" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [ - { - "name": "ProductID", - "type": "int", - "precision": 10 - }, - { - "name": "Name", - "type": "nvarchar" - }, - { - "name": "ProductNumber", - "type": "nvarchar" - }, - { - "name": "Color", - "type": "nvarchar" - }, - { - "name": "StandardCost", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "ListPrice", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "Size", - "type": "nvarchar" - }, - { - "name": "Weight", - "type": "decimal", - "precision": 8, - "scale": 2 - }, - { - "name": "ProductCategoryID", - "type": "int", - "precision": 10 - }, - { - "name": "ProductModelID", - "type": "int", - "precision": 10 - }, - { - "name": "SellStartDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "SellEndDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "DiscontinuedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "ThumbNailPhoto", - "type": "varbinary" - }, - { - "name": "ThumbnailPhotoFileName", - "type": "nvarchar" - }, - { - "name": "rowguid", - "type": "uniqueidentifier" - }, - { - "name": "ModifiedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - } - ], - "typeProperties": { - "schema": "SalesLT", - "table": "Product" - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ASQL_ProductModel.json b/Code/DataFactory/dataset/ASQL_ProductModel.json deleted file mode 100644 index be402d9..0000000 --- a/Code/DataFactory/dataset/ASQL_ProductModel.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "name": "ASQL_ProductModel", - "properties": { - "linkedServiceName": { - "referenceName": "AzureSqlDatabase1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [ - { - "name": "ProductID", - "type": "int", - "precision": 10 - }, - { - "name": "Name", - "type": "nvarchar" - }, - { - "name": "ProductNumber", - "type": "nvarchar" - }, - { - "name": "Color", - "type": "nvarchar" - }, - { - "name": "StandardCost", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "ListPrice", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "Size", - "type": "nvarchar" - }, - { - "name": "Weight", - "type": "decimal", - "precision": 8, - "scale": 2 - }, - { - "name": "ProductCategoryID", - "type": "int", - "precision": 10 - }, - { - "name": "ProductModelID", - "type": "int", - "precision": 10 - }, - { - "name": "SellStartDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "SellEndDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "DiscontinuedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "ThumbNailPhoto", - "type": "varbinary" - }, - { - "name": "ThumbnailPhotoFileName", - "type": "nvarchar" - }, - { - "name": "rowguid", - "type": "uniqueidentifier" - }, - { - "name": "ModifiedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - } - ], - "typeProperties": { - "schema": "SalesLT", - "table": "ProductModel" - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json b/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json new file mode 100644 index 0000000..139ac39 --- /dev/null +++ b/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json @@ -0,0 +1,19 @@ +{ + "name": "HTTP_BIN_AWProduct", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWProduct", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Labs" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "HttpServerLocation" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json b/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json new file mode 100644 index 0000000..039b3cd --- /dev/null +++ b/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json @@ -0,0 +1,28 @@ +{ + "name": "HTTP_BIN_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWGitHub", + "type": "LinkedServiceReference" + }, + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Labs" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "HttpServerLocation", + "relativeUrl": { + "value": "@{dataset().Entity}.csv", + "type": "Expression" + } + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json b/Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json new file mode 100644 index 0000000..6770d8a --- /dev/null +++ b/Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json @@ -0,0 +1,18 @@ +{ + "name": "ADLS_saintegrationpipelines", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "https://traininglake01.dfs.core.windows.net", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "traininglake01" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/HTTP_AWGitHub.json b/Code/DataFactory/linkedService/HTTP_AWGitHub.json new file mode 100644 index 0000000..ee6b6c3 --- /dev/null +++ b/Code/DataFactory/linkedService/HTTP_AWGitHub.json @@ -0,0 +1,12 @@ +{ + "name": "HTTP_AWGitHub", + "properties": { + "annotations": [], + "type": "HttpServer", + "typeProperties": { + "url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/", + "enableServerCertificateValidation": true, + "authenticationType": "Anonymous" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/HTTP_AWProduct.json b/Code/DataFactory/linkedService/HTTP_AWProduct.json new file mode 100644 index 0000000..d5f1697 --- /dev/null +++ b/Code/DataFactory/linkedService/HTTP_AWProduct.json @@ -0,0 +1,12 @@ +{ + "name": "HTTP_AWProduct", + "properties": { + "annotations": [], + "type": "HttpServer", + "typeProperties": { + "url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv", + "enableServerCertificateValidation": true, + "authenticationType": "Anonymous" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/BuildDimProduct.json b/Code/DataFactory/pipeline/BuildDimProduct.json new file mode 100644 index 0000000..34a2e03 --- /dev/null +++ b/Code/DataFactory/pipeline/BuildDimProduct.json @@ -0,0 +1,46 @@ +{ + "name": "BuildDimProduct", + "properties": { + "activities": [ + { + "name": "Run UpdateProductDimension", + "type": "ExecuteDataFlow", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataflow": { + "referenceName": "UpdateProductDimension", + "type": "DataFlowReference", + "datasetParameters": { + "Product": { + "FileName": "Product.tsv" + }, + "ProductSubcategory": { + "FileName": "ProductSubcategory.tsv" + }, + "ProductCategory": { + "FileName": "ProductCategory.tsv" + } + } + }, + "compute": { + "coreCount": 8, + "computeType": "General" + }, + "traceLevel": "Fine" + } + } + ], + "folder": { + "name": "Labs" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/BuildProductDimension.json b/Code/DataFactory/pipeline/BuildProductDimension.json deleted file mode 100644 index ff36156..0000000 --- a/Code/DataFactory/pipeline/BuildProductDimension.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "BuildProductDimension", - "properties": { - "activities": [ - { - "name": "BuildProductDimension", - "type": "ExecuteDataFlow", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "dataflow": { - "referenceName": "BuildProductDimension", - "type": "DataFlowReference" - }, - "compute": { - "coreCount": 8, - "computeType": "General" - } - } - } - ], - "folder": { - "name": "Lab4" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T17:05:06Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAWEntities.json b/Code/DataFactory/pipeline/CopyAWEntities.json new file mode 100644 index 0000000..ad648d9 --- /dev/null +++ b/Code/DataFactory/pipeline/CopyAWEntities.json @@ -0,0 +1,55 @@ +{ + "name": "CopyAWEntities", + "properties": { + "activities": [ + { + "name": "Copy each AW entity", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@variables('Entities')", + "type": "Expression" + }, + "isSequential": false, + "activities": [ + { + "name": "Execute CopyAWEntity", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "CopyAWEntity", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "Entity": { + "value": "@item()", + "type": "Expression" + } + } + } + } + ] + } + } + ], + "variables": { + "Entities": { + "type": "Array", + "defaultValue": [ + "Product", + "ProductSubcategory", + "ProductCategory" + ] + } + }, + "folder": { + "name": "Labs" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAWEntity.json b/Code/DataFactory/pipeline/CopyAWEntity.json new file mode 100644 index 0000000..5a0db2d --- /dev/null +++ b/Code/DataFactory/pipeline/CopyAWEntity.json @@ -0,0 +1,72 @@ +{ + "name": "CopyAWEntity", + "properties": { + "activities": [ + { + "name": "Copy AW entity", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "HTTP_BIN_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "Entity": { + "value": "@pipeline().parameters.Entity", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLS_BIN_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "Entity": { + "value": "@pipeline().parameters.Entity", + "type": "Expression" + } + } + } + ] + } + ], + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Labs" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAWProduct.json b/Code/DataFactory/pipeline/CopyAWProduct.json new file mode 100644 index 0000000..1d11002 --- /dev/null +++ b/Code/DataFactory/pipeline/CopyAWProduct.json @@ -0,0 +1,55 @@ +{ + "name": "CopyAWProduct", + "properties": { + "activities": [ + { + "name": "Copy AW Products to data lake", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "HTTP_BIN_AWProduct", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ADLS_BIN_AWProduct", + "type": "DatasetReference" + } + ] + } + ], + "folder": { + "name": "Labs" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAdventureWorks.json b/Code/DataFactory/pipeline/CopyAdventureWorks.json deleted file mode 100644 index cf3a0b6..0000000 --- a/Code/DataFactory/pipeline/CopyAdventureWorks.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "name": "CopyAdventureWorks", - "properties": { - "activities": [ - { - "name": "Get table list", - "type": "Lookup", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "sqlReaderQuery": "SELECT TABLE_SCHEMA, TABLE_NAME \nFROM INFORMATION_SCHEMA.TABLES\nWHERE TABLE_TYPE = 'BASE TABLE'\nAND TABLE_SCHEMA = 'SalesLT'\n", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "dataset": { - "referenceName": "ASQL_Product", - "type": "DatasetReference" - }, - "firstRowOnly": false - } - }, - { - "name": "ForEach1", - "type": "ForEach", - "dependsOn": [ - { - "activity": "Get table list", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "items": { - "value": "@activity('Get table list').output.value", - "type": "Expression" - }, - "activities": [ - { - "name": "Execute Pipeline1", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "CopyAnyTable", - "type": "PipelineReference" - }, - "waitOnCompletion": true, - "parameters": { - "TableSchema": { - "value": "@item().TABLE_SCHEMA", - "type": "Expression" - }, - "TableName": { - "value": "@item().TABLE_NAME", - "type": "Expression" - } - } - } - } - ] - } - } - ], - "folder": { - "name": "Lab5" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T17:11:52Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAnyTable.json b/Code/DataFactory/pipeline/CopyAnyTable.json deleted file mode 100644 index 2992669..0000000 --- a/Code/DataFactory/pipeline/CopyAnyTable.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "name": "CopyAnyTable", - "properties": { - "activities": [ - { - "name": "Copy table", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "ASQL_AdventureWorks_Generic", - "type": "DatasetReference", - "parameters": { - "TableSchema": { - "value": "@pipeline().parameters.TableSchema", - "type": "Expression" - }, - "TableName": { - "value": "@pipeline().parameters.TableName", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "ADLS_Raw_Generic", - "type": "DatasetReference", - "parameters": { - "File": { - "value": "@{pipeline().parameters.TableSchema}.@{pipeline().parameters.TableName}.txt", - "type": "Expression" - } - } - } - ] - } - ], - "parameters": { - "TableSchema": { - "type": "string" - }, - "TableName": { - "type": "string" - } - }, - "folder": { - "name": "Lab5" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T17:11:52Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyProduct.json b/Code/DataFactory/pipeline/CopyProduct.json deleted file mode 100644 index 70cae29..0000000 --- a/Code/DataFactory/pipeline/CopyProduct.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "CopyProduct", - "properties": { - "activities": [ - { - "name": "Copy data to lake", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "ASQL_Product", - "type": "DatasetReference" - } - ], - "outputs": [ - { - "referenceName": "ADLS_Raw", - "type": "DatasetReference" - } - ] - } - ], - "folder": { - "name": "Lab2" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T14:33:50Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyProductCategory.json b/Code/DataFactory/pipeline/CopyProductCategory.json deleted file mode 100644 index 08290d9..0000000 --- a/Code/DataFactory/pipeline/CopyProductCategory.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "CopyProductCategory", - "properties": { - "activities": [ - { - "name": "Execute CopyProductCategory", - "type": "ExecuteSSISPackage", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "packageLocation": { - "packagePath": "ADayFullOfAdf/SsisPackages/CopyProductCategory.dtsx", - "type": "SSISDB" - }, - "environmentPath": null, - "connectVia": { - "referenceName": "SSISIntegrationRuntime", - "type": "IntegrationRuntimeReference" - }, - "loggingLevel": "Basic" - } - } - ], - "folder": { - "name": "Lab3" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T15:27:22Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyProductModel.json b/Code/DataFactory/pipeline/CopyProductModel.json deleted file mode 100644 index d19be52..0000000 --- a/Code/DataFactory/pipeline/CopyProductModel.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "CopyProductModel", - "properties": { - "activities": [ - { - "name": "Copy data1", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "ASQL_ProductModel", - "type": "DatasetReference" - } - ], - "outputs": [ - { - "referenceName": "ADLS_Raw", - "type": "DatasetReference" - } - ] - } - ], - "folder": { - "name": "Lab4" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T16:27:34Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Labs/Lab 1 - Create Azure resources.pdf b/Labs/Lab 1 - Create Azure resources.pdf index eba1ac3..58c8690 100644 Binary files a/Labs/Lab 1 - Create Azure resources.pdf and b/Labs/Lab 1 - Create Azure resources.pdf differ diff --git a/Labs/Lab 2 - Build a Copy data pipeline.pdf b/Labs/Lab 2 - Build a Copy data pipeline.pdf deleted file mode 100644 index 2ca260b..0000000 Binary files a/Labs/Lab 2 - Build a Copy data pipeline.pdf and /dev/null differ diff --git a/Labs/Lab 2 - Build a copy pipeline.pdf b/Labs/Lab 2 - Build a copy pipeline.pdf new file mode 100644 index 0000000..6771d58 Binary files /dev/null and b/Labs/Lab 2 - Build a copy pipeline.pdf differ diff --git a/Labs/Lab 3 - Create a reusable pipeline.pdf b/Labs/Lab 3 - Create a reusable pipeline.pdf new file mode 100644 index 0000000..1637fa0 Binary files /dev/null and b/Labs/Lab 3 - Create a reusable pipeline.pdf differ diff --git a/Labs/Lab 3 - Run SSIS packages in ADF.pdf b/Labs/Lab 3 - Run SSIS packages in ADF.pdf deleted file mode 100644 index 5b890ef..0000000 Binary files a/Labs/Lab 3 - Run SSIS packages in ADF.pdf and /dev/null differ diff --git a/Labs/Lab 4 - Author a data flow.pdf b/Labs/Lab 4 - Author a data flow.pdf new file mode 100644 index 0000000..0ad1139 Binary files /dev/null and b/Labs/Lab 4 - Author a data flow.pdf differ diff --git a/Labs/Lab 4 - Build a Mapping Data Flow.pdf b/Labs/Lab 4 - Build a Mapping Data Flow.pdf deleted file mode 100644 index c1a7421..0000000 Binary files a/Labs/Lab 4 - Build a Mapping Data Flow.pdf and /dev/null differ diff --git a/Labs/Lab 5 - Create a reusable pipeline.pdf b/Labs/Lab 5 - Create a reusable pipeline.pdf deleted file mode 100644 index 9fc6619..0000000 Binary files a/Labs/Lab 5 - Create a reusable pipeline.pdf and /dev/null differ diff --git a/Labs/Lab 5 - Monitor factory activity.pdf b/Labs/Lab 5 - Monitor factory activity.pdf new file mode 100644 index 0000000..93fd6e9 Binary files /dev/null and b/Labs/Lab 5 - Monitor factory activity.pdf differ diff --git a/Labs/Lab 6 - Explore Synapse pipelines.pdf b/Labs/Lab 6 - Explore Synapse pipelines.pdf new file mode 100644 index 0000000..22a6527 Binary files /dev/null and b/Labs/Lab 6 - Explore Synapse pipelines.pdf differ diff --git a/Labs/Lab 6 - Monitor factory activity.pdf b/Labs/Lab 6 - Monitor factory activity.pdf deleted file mode 100644 index 73e14a3..0000000 Binary files a/Labs/Lab 6 - Monitor factory activity.pdf and /dev/null differ