diff --git a/Code/DataFactory/dataflow/MappingOrderAggregationWithParam.json b/Code/DataFactory/dataflow/MappingOrderAggregationWithParam.json new file mode 100644 index 0000000..d584639 --- /dev/null +++ b/Code/DataFactory/dataflow/MappingOrderAggregationWithParam.json @@ -0,0 +1,45 @@ +{ + "name": "MappingOrderAggregationWithParam", + "properties": { + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "LakeFileOrderHeaderParquet", + "type": "DatasetReference" + }, + "name": "OrderHeader" + }, + { + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + }, + "name": "OrderLineDetails" + } + ], + "sinks": [ + { + "dataset": { + "referenceName": "TableOrderSummary", + "type": "DatasetReference" + }, + "name": "OrderSummary" + } + ], + "transformations": [ + { + "name": "JoinHeaderToLineDetails" + }, + { + "name": "OrderLineCount" + }, + { + "name": "AddAuditColum" + } + ], + "script": "parameters{\n\tAuditColumn as string\n}\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tmatchType:'exact',\n\tignoreSpaces: false,\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount derive(AuditValue = $AuditColumn) ~> AddAuditColum\nAddAuditColum sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json b/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json index 13f0ffd..edccc77 100644 --- a/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json +++ b/Code/DataFactory/dataset/ADLS_BIN_AWProduct.json @@ -6,7 +6,7 @@ "type": "LinkedServiceReference" }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", diff --git a/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json b/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json index 934edf6..b0006b8 100644 --- a/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json +++ b/Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json @@ -11,7 +11,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", diff --git a/Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json b/Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json new file mode 100644 index 0000000..53eef04 --- /dev/null +++ b/Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json @@ -0,0 +1,32 @@ +{ + "name": "ADLS_PQT_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "ADLS_saintegrationpipelines", + "type": "LinkedServiceReference" + }, + "parameters": { + "EntityName": { + "type": "string" + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@{dataset().EntityName}.parquet", + "type": "Expression" + }, + "folderPath": "Raw", + "fileSystem": "lakeroot" + }, + "compressionCodec": "snappy" + }, + "schema": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json b/Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json index c23301b..ca0f4e3 100644 --- a/Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json +++ b/Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json @@ -11,7 +11,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "DelimitedText", diff --git a/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json b/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json index 139ac39..d71545c 100644 --- a/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json +++ b/Code/DataFactory/dataset/HTTP_BIN_AWProduct.json @@ -6,7 +6,7 @@ "type": "LinkedServiceReference" }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", diff --git a/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json b/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json index 039b3cd..ece8886 100644 --- a/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json +++ b/Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json @@ -11,7 +11,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [], "type": "Binary", diff --git a/Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json b/Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json new file mode 100644 index 0000000..ac8a75a --- /dev/null +++ b/Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json @@ -0,0 +1,32 @@ +{ + "name": "HTTP_TSV_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWGitHub", + "type": "LinkedServiceReference" + }, + "parameters": { + "EntityName": { + "type": "string" + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "HttpServerLocation", + "relativeUrl": { + "value": "@{dataset().EntityName}.csv", + "type": "Expression" + } + }, + "columnDelimiter": "\t", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/factory/TrainingFactoryDev.json b/Code/DataFactory/factory/TrainingFactoryDev.json index 56d7c13..d6ad793 100644 --- a/Code/DataFactory/factory/TrainingFactoryDev.json +++ b/Code/DataFactory/factory/TrainingFactoryDev.json @@ -10,6 +10,9 @@ "type": "string", "value": "trainingkeys01" } + }, + "globalConfigurations": { + "PipelineBillingEnabled": "true" } }, "location": "uksouth", diff --git a/Code/DataFactory/linkedService/EmailSenderFunction.json b/Code/DataFactory/linkedService/EmailSenderFunction.json new file mode 100644 index 0000000..2dd3def --- /dev/null +++ b/Code/DataFactory/linkedService/EmailSenderFunction.json @@ -0,0 +1,13 @@ +{ + "name": "EmailSenderFunction", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureFunction", + "typeProperties": { + "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", + "authentication": "Anonymous", + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9jZGYyMmI4Ni0zNTk2LTRkNzktYjdmOC1kNGNiMjUyNDZmYjUiDQp9" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/BuildDimProduct.json b/Code/DataFactory/pipeline/BuildDimProduct.json index 34a2e03..2e8789b 100644 --- a/Code/DataFactory/pipeline/BuildDimProduct.json +++ b/Code/DataFactory/pipeline/BuildDimProduct.json @@ -39,7 +39,7 @@ } ], "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] } diff --git a/Code/DataFactory/pipeline/CopyAWEntities.json b/Code/DataFactory/pipeline/CopyAWEntities.json index ad648d9..ac9a59d 100644 --- a/Code/DataFactory/pipeline/CopyAWEntities.json +++ b/Code/DataFactory/pipeline/CopyAWEntities.json @@ -48,7 +48,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] } diff --git a/Code/DataFactory/pipeline/CopyAWEntity.json b/Code/DataFactory/pipeline/CopyAWEntity.json index 5a0db2d..004f1ff 100644 --- a/Code/DataFactory/pipeline/CopyAWEntity.json +++ b/Code/DataFactory/pipeline/CopyAWEntity.json @@ -65,7 +65,7 @@ } }, "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] } diff --git a/Code/DataFactory/pipeline/CopyAWProduct.json b/Code/DataFactory/pipeline/CopyAWProduct.json index 1d11002..d51cdf7 100644 --- a/Code/DataFactory/pipeline/CopyAWProduct.json +++ b/Code/DataFactory/pipeline/CopyAWProduct.json @@ -48,7 +48,7 @@ } ], "folder": { - "name": "Labs" + "name": "Labs1" }, "annotations": [] } diff --git a/Code/DataFactory/pipeline/CopyAWUsingMetadata.json b/Code/DataFactory/pipeline/CopyAWUsingMetadata.json new file mode 100644 index 0000000..b3d2a03 --- /dev/null +++ b/Code/DataFactory/pipeline/CopyAWUsingMetadata.json @@ -0,0 +1,170 @@ +{ + "name": "CopyAWUsingMetadata", + "properties": { + "activities": [ + { + "name": "Get catalog", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": "https://raw.githubusercontent.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training/main/Labs/TableCatalog.json", + "method": "GET" + } + }, + { + "name": "Cache catalog as JSON array", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get catalog", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "Catalog", + "value": { + "value": "@json(activity('Get catalog').output.Response)", + "type": "Expression" + } + } + }, + { + "name": "ForEach AW entity", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Cache catalog as JSON array", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@variables('Catalog')", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy AW entity", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "ParquetSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "ParquetWriteSettings" + } + }, + "enableStaging": false, + "translator": { + "value": "@item().translator", + "type": "Expression" + } + }, + "inputs": [ + { + "referenceName": "HTTP_TSV_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "EntityName": { + "value": "@item().EntityName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLS_PQT_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "EntityName": { + "value": "@item().EntityName", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Notify error", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Copy AW entity", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "SendEmail", + "method": "POST", + "body": { + "value": "{\n \"emailRecipients\": \"richard.swinbank@gmail.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "EmailSenderFunction", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "variables": { + "Catalog": { + "type": "Array", + "defaultValue": [] + } + }, + "folder": { + "name": "Labs2" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Custom Activity.json b/Code/DataFactory/pipeline/Custom Activity.json index f04aff1..0bc5625 100644 --- a/Code/DataFactory/pipeline/Custom Activity.json +++ b/Code/DataFactory/pipeline/Custom Activity.json @@ -30,35 +30,6 @@ "referenceName": "BatchForTraining01", "type": "LinkedServiceReference" } - }, - { - "name": "HelloWorld - Again", - "type": "Custom", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "command": "CustomActivity.exe", - "resourceLinkedService": { - "referenceName": "TrainingStore01", - "type": "LinkedServiceReference" - }, - "folderPath": "customactivities", - "referenceObjects": { - "linkedServices": [], - "datasets": [] - } - }, - "linkedServiceName": { - "referenceName": "BatchForTraining01", - "type": "LinkedServiceReference" - } } ], "folder": { diff --git a/Code/DataFactory/pipeline/EmailSender.json b/Code/DataFactory/pipeline/EmailSender.json new file mode 100644 index 0000000..679db2f --- /dev/null +++ b/Code/DataFactory/pipeline/EmailSender.json @@ -0,0 +1,51 @@ +{ + "name": "EmailSender", + "properties": { + "activities": [ + { + "name": "Send Email", + "type": "AzureFunctionActivity", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "SendEmail", + "method": "POST", + "body": { + "value": "@concat('\n{\n\"emailRecipients\": \"',pipeline().parameters.Recipients,'\",\n\"emailSubject\": \"',pipeline().parameters.Subject,'\",\n\"emailBody\": \"',pipeline().parameters.Body,'\",\n\"emailImportance\": \"',pipeline().parameters.Importance,'\"\n}')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "EmailSenderFunction", + "type": "LinkedServiceReference" + } + } + ], + "parameters": { + "Recipients": { + "type": "string" + }, + "Subject": { + "type": "string" + }, + "Body": { + "type": "string" + }, + "Importance": { + "type": "string", + "defaultValue": "High" + } + }, + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Order Summary with Mapping With Param.json b/Code/DataFactory/pipeline/Order Summary with Mapping With Param.json new file mode 100644 index 0000000..6347c62 --- /dev/null +++ b/Code/DataFactory/pipeline/Order Summary with Mapping With Param.json @@ -0,0 +1,48 @@ +{ + "name": "Order Summary with Mapping With Param", + "properties": { + "activities": [ + { + "name": "Mapping Order Aggregation", + "type": "ExecuteDataFlow", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataflow": { + "referenceName": "MappingOrderAggregationWithParam", + "type": "DataFlowReference", + "parameters": { + "AuditColumn": { + "value": "'@{pipeline().parameters.AuditColumnValue}'", + "type": "Expression" + } + } + }, + "compute": { + "coreCount": 8, + "computeType": "General" + }, + "traceLevel": "Fine" + } + } + ], + "parameters": { + "AuditColumnValue": { + "type": "string" + } + }, + "folder": { + "name": "Demo Pipelines/Data Flows" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:15Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Labs/Lab 7 - Mini-project.pdf b/Labs/Lab 7 - Mini-project.pdf new file mode 100644 index 0000000..d8190d9 Binary files /dev/null and b/Labs/Lab 7 - Mini-project.pdf differ diff --git a/Labs/TableCatalog.json b/Labs/TableCatalog.json new file mode 100644 index 0000000..8a66497 --- /dev/null +++ b/Labs/TableCatalog.json @@ -0,0 +1,1082 @@ +[ + { + "entityName": "Product", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 2 + }, + "sink": { + "name": "Name", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 3 + }, + "sink": { + "name": "ProductNumber", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 4 + }, + "sink": { + "name": "MakeFlag", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 5 + }, + "sink": { + "name": "FinishedGoodsFlag", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 6 + }, + "sink": { + "name": "Color", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 7 + }, + "sink": { + "name": "SafetyStockLevel", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 8 + }, + "sink": { + "name": "ReorderPoint", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 9 + }, + "sink": { + "name": "StandardCost", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 10 + }, + "sink": { + "name": "ListPrice", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 11 + }, + "sink": { + "name": "Size", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 12 + }, + "sink": { + "name": "SizeUnitMeasureCode", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 13 + }, + "sink": { + "name": "WeightUnitMeasureCode", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 14 + }, + "sink": { + "name": "Weight", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 15 + }, + "sink": { + "name": "DaysToManufacture", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 16 + }, + "sink": { + "name": "ProductLine", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 17 + }, + "sink": { + "name": "Class", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 18 + }, + "sink": { + "name": "Style", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 19 + }, + "sink": { + "name": "ProductSubcategoryID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 20 + }, + "sink": { + "name": "ProductModelID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 21 + }, + "sink": { + "name": "SellStartDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 22 + }, + "sink": { + "name": "SellEndDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 23 + }, + "sink": { + "name": "DiscontinuedDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Guid", + "ordinal": 24 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 25 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductCategory", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductCategoryID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 2 + }, + "sink": { + "name": "Name", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Guid", + "ordinal": 3 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 4 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductCostHistory", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 2 + }, + "sink": { + "name": "StartDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 3 + }, + "sink": { + "name": "EndDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 4 + }, + "sink": { + "name": "StandardCost", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 5 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductDescription", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductDescriptionID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 2 + }, + "sink": { + "name": "Description", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Guid", + "ordinal": 3 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 4 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductDocument", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 2 + }, + "sink": { + "name": "DocumentNode", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 3 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductInventory", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 2 + }, + "sink": { + "name": "LocationID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 3 + }, + "sink": { + "name": "Shelf", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 4 + }, + "sink": { + "name": "Bin", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 5 + }, + "sink": { + "name": "Quantity", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 6 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 7 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductListPriceHistory", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 2 + }, + "sink": { + "name": "StartDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 3 + }, + "sink": { + "name": "EndDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 4 + }, + "sink": { + "name": "ListPrice", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 5 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "ProductSubcategory", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "ProductSubcategoryID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 2 + }, + "sink": { + "name": "ProductCategoryID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 3 + }, + "sink": { + "name": "Name", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Guid", + "ordinal": 4 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 5 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "SalesOrderDetail", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "SalesOrderID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 2 + }, + "sink": { + "name": "SalesOrderDetailID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 3 + }, + "sink": { + "name": "CarrierTrackingNumber", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 4 + }, + "sink": { + "name": "OrderQty", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 5 + }, + "sink": { + "name": "ProductID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 6 + }, + "sink": { + "name": "SpecialOfferID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 7 + }, + "sink": { + "name": "UnitPrice", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 8 + }, + "sink": { + "name": "UnitPriceDiscount", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 9 + }, + "sink": { + "name": "LineTotal", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Guid", + "ordinal": 10 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 11 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + { + "entityName": "SalesOrderHeader", + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "type": "Int32", + "ordinal": 1 + }, + "sink": { + "name": "SalesOrderID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 2 + }, + "sink": { + "name": "RevisionNumber", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 3 + }, + "sink": { + "name": "OrderDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 4 + }, + "sink": { + "name": "DueDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 5 + }, + "sink": { + "name": "ShipDate", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 6 + }, + "sink": { + "name": "Status", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 7 + }, + "sink": { + "name": "OnlineOrderFlag", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 8 + }, + "sink": { + "name": "SalesOrderNumber", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 9 + }, + "sink": { + "name": "PurchaseOrderNumber", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 10 + }, + "sink": { + "name": "AccountNumber", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 11 + }, + "sink": { + "name": "CustomerID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 12 + }, + "sink": { + "name": "SalesPersonID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 13 + }, + "sink": { + "name": "TerritoryID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 14 + }, + "sink": { + "name": "BillToAddressID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 15 + }, + "sink": { + "name": "ShipToAddressID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 16 + }, + "sink": { + "name": "ShipMethodID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 17 + }, + "sink": { + "name": "CreditCardID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 18 + }, + "sink": { + "name": "CreditCardApprovalCode", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Int32", + "ordinal": 19 + }, + "sink": { + "name": "CurrencyRateID", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 20 + }, + "sink": { + "name": "SubTotal", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 21 + }, + "sink": { + "name": "TaxAmt", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 22 + }, + "sink": { + "name": "Freight", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Decimal", + "ordinal": 23 + }, + "sink": { + "name": "TotalDue", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "String", + "ordinal": 24 + }, + "sink": { + "name": "Comment", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "Guid", + "ordinal": 25 + }, + "sink": { + "name": "rowguid", + "physicalType": "UTF8" + } + }, + { + "source": { + "type": "DateTime", + "ordinal": 26 + }, + "sink": { + "name": "ModifiedDate", + "physicalType": "UTF8" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + } +] \ No newline at end of file