64 lines
3.7 KiB
JSON
64 lines
3.7 KiB
JSON
{
|
|
"name": "BuildProductDimension",
|
|
"properties": {
|
|
"folder": {
|
|
"name": "Lab4"
|
|
},
|
|
"type": "MappingDataFlow",
|
|
"typeProperties": {
|
|
"sources": [
|
|
{
|
|
"dataset": {
|
|
"referenceName": "ADLS_Product",
|
|
"type": "DatasetReference"
|
|
},
|
|
"name": "Product"
|
|
},
|
|
{
|
|
"dataset": {
|
|
"referenceName": "ADLS_ProductCategory",
|
|
"type": "DatasetReference"
|
|
},
|
|
"name": "ProductCategory"
|
|
},
|
|
{
|
|
"dataset": {
|
|
"referenceName": "ADLS_ProductModel",
|
|
"type": "DatasetReference"
|
|
},
|
|
"name": "ProductModel"
|
|
}
|
|
],
|
|
"sinks": [
|
|
{
|
|
"dataset": {
|
|
"referenceName": "ADLS_DimProduct",
|
|
"type": "DatasetReference"
|
|
},
|
|
"name": "DimProduct"
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"name": "SelectProductColumns"
|
|
},
|
|
{
|
|
"name": "SelectProductCategoryColumns"
|
|
},
|
|
{
|
|
"name": "SelectProductModelColumns"
|
|
},
|
|
{
|
|
"name": "LookupProductCategory"
|
|
},
|
|
{
|
|
"name": "LookupProductModelID"
|
|
},
|
|
{
|
|
"name": "RemoveDuplicateColumns"
|
|
}
|
|
],
|
|
"script": "source(output(\n\t\tProductID as short,\n\t\tName as string,\n\t\tProductNumber as string,\n\t\tColor as string,\n\t\tStandardCost as double,\n\t\tListPrice as double,\n\t\tSize as string,\n\t\tWeight as double,\n\t\tProductCategoryID as short,\n\t\tProductModelID as short,\n\t\tSellStartDate as string,\n\t\tSellEndDate as string,\n\t\tDiscontinuedDate as string,\n\t\tThumbNailPhoto as string,\n\t\tThumbnailPhotoFileName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> Product\nsource(output(\n\t\tProductCategoryID as short,\n\t\tParentProductCategoryID as string,\n\t\tName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductCategory\nsource(output(\n\t\tProductModelID as short,\n\t\tName as string,\n\t\tCatalogDescription as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductModel\nProduct select(mapColumn(\n\t\tProductID,\n\t\tProduct = Name,\n\t\tProductCategoryID,\n\t\tProductModelID\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductColumns\nProductCategory select(mapColumn(\n\t\tProductCategoryID,\n\t\tProductCategory = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns\nProductModel select(mapColumn(\n\t\tProductModelID,\n\t\tProductModel = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductModelColumns\nSelectProductColumns, SelectProductCategoryColumns lookup(SelectProductColumns@ProductCategoryID == SelectProductCategoryColumns@ProductCategoryID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductCategory\nLookupProductCategory, SelectProductModelColumns lookup(SelectProductColumns@ProductModelID == SelectProductModelColumns@ProductModelID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductModelID\nLookupProductModelID select(mapColumn(\n\t\tProductID,\n\t\tProduct,\n\t\tProductCategoryID = {SelectProductColumns@ProductCategoryID},\n\t\tProductModelID = {SelectProductColumns@ProductModelID},\n\t\tProductCategory,\n\t\tProductModel\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns\nRemoveDuplicateColumns sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'parquet',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> DimProduct"
|
|
}
|
|
}
|
|
} |