diff --git a/Code/DataFactory/integrationRuntime/AzureVM.json b/Code/DataFactory/integrationRuntime/AzureVM.json new file mode 100644 index 0000000..050d0f8 --- /dev/null +++ b/Code/DataFactory/integrationRuntime/AzureVM.json @@ -0,0 +1,6 @@ +{ + "name": "AzureVM", + "properties": { + "type": "SelfHosted" + } +} \ No newline at end of file diff --git a/Code/DataFactory/integrationRuntime/Laptop.json b/Code/DataFactory/integrationRuntime/Laptop.json new file mode 100644 index 0000000..3df4c7a --- /dev/null +++ b/Code/DataFactory/integrationRuntime/Laptop.json @@ -0,0 +1,6 @@ +{ + "name": "Laptop", + "properties": { + "type": "SelfHosted" + } +} \ No newline at end of file diff --git a/Code/DataFactory/integrationRuntime/PaulsLaptopIR.json b/Code/DataFactory/integrationRuntime/PaulsLaptopIR.json deleted file mode 100644 index a19ae86..0000000 --- a/Code/DataFactory/integrationRuntime/PaulsLaptopIR.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "PaulsLaptopIR", - "properties": { - "type": "SelfHosted", - "typeProperties": { - "linkedInfo": { - "resourceId": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WINAP71rY2YabRV", - "authorizationType": "Rbac" - } - } - } -} \ No newline at end of file diff --git a/Code/DataFactory/integrationRuntime/SSISIntegrationRuntime.json b/Code/DataFactory/integrationRuntime/SSISIntegrationRuntime.json deleted file mode 100644 index 079ebe3..0000000 --- a/Code/DataFactory/integrationRuntime/SSISIntegrationRuntime.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "SSISIntegrationRuntime", - "properties": { - "type": "Managed", - "typeProperties": { - "computeProperties": { - "location": "UK South", - "nodeSize": "Standard_D2_v3", - "numberOfNodes": 1, - "maxParallelExecutionsPerNode": 2 - }, - "ssisProperties": { - "catalogInfo": { - "catalogServerEndpoint": "adayfullofadf-sql.database.windows.net", - "catalogAdminUserName": "sql-admin", - "catalogAdminPassword": { - "type": "SecureString", - "value": "**********" - }, - "catalogPricingTier": "S1" - }, - "edition": "Standard", - "licenseType": "LicenseIncluded" - } - } - } -} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/LaptopFiles.json b/Code/DataFactory/linkedService/LaptopFiles.json index 7ab03b8..446638c 100644 --- a/Code/DataFactory/linkedService/LaptopFiles.json +++ b/Code/DataFactory/linkedService/LaptopFiles.json @@ -5,19 +5,12 @@ "annotations": [], "type": "FileServer", "typeProperties": { - "host": "C:\\ADFRoot\\", - "userId": "ADFIRUser", - "password": { - "type": "AzureKeyVaultSecret", - "store": { - "referenceName": "TrainingKeys01", - "type": "LinkedServiceReference" - }, - "secretName": "ADFIRUser" - } + "host": "C:\\ADFRoot", + "userId": "paul.andrew@avanade.com", + "encryptedCredential": "eyJDcmVkZW50aWFsSWQiOiIzMDg5NjgxZi05MmM0LTQzMzAtYjMzYy02ZTdhNjA3ZWUxZTkiLCJWZXJzaW9uIjoiMi4wIiwiQ2xhc3NUeXBlIjoiTWljcm9zb2Z0LkRhdGFQcm94eS5Db3JlLkludGVyU2VydmljZURhdGFDb250cmFjdC5DcmVkZW50aWFsU1UwNkNZMTQifQ==" }, "connectVia": { - "referenceName": "PaulsLaptopIR", + "referenceName": "Laptop", "type": "IntegrationRuntimeReference" } } diff --git a/Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureFunctionTest.json b/Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureFunctionTest.json new file mode 100644 index 0000000..e58fe03 --- /dev/null +++ b/Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureFunctionTest.json @@ -0,0 +1,7 @@ +{ + "name": "AzureFunctionTest", + "properties": { + "privateLinkResourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions", + "groupId": "sites" + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAWUsingMetadata.json b/Code/DataFactory/pipeline/CopyAWUsingMetadata.json index b3d2a03..d2af86c 100644 --- a/Code/DataFactory/pipeline/CopyAWUsingMetadata.json +++ b/Code/DataFactory/pipeline/CopyAWUsingMetadata.json @@ -143,7 +143,7 @@ "functionName": "SendEmail", "method": "POST", "body": { - "value": "{\n \"emailRecipients\": \"richard.swinbank@gmail.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}", + "value": "{\n \"emailRecipients\": \"paul@mrpaulandrew.com\",\n \"emailSubject\": \"Error copying @{item().EntityName}\",\n \"emailBody\": \"@{replace(activity('Copy AW entity').output.errors[0].Message,'\"','\\\"')}\"\n}", "type": "Expression" } }, diff --git a/Code/DataFactory/trigger/trigger1.json b/Code/DataFactory/trigger/trigger1.json new file mode 100644 index 0000000..6f84eeb --- /dev/null +++ b/Code/DataFactory/trigger/trigger1.json @@ -0,0 +1,17 @@ +{ + "name": "trigger1", + "properties": { + "annotations": [], + "runtimeState": "Stopped", + "pipelines": [], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Minute", + "interval": 15, + "startTime": "2023-02-06T14:56:00Z", + "timeZone": "UTC" + } + } + } +} \ No newline at end of file diff --git a/Labs/Lab 1 - Create Azure resources.pdf b/Labs/Lab 1 - Create Azure resources.pdf index 58c8690..9d03648 100644 Binary files a/Labs/Lab 1 - Create Azure resources.pdf and b/Labs/Lab 1 - Create Azure resources.pdf differ diff --git a/Labs/Lab 2 - Build a copy pipeline.pdf b/Labs/Lab 2 - Build a copy pipeline.pdf index 6771d58..bdf4103 100644 Binary files a/Labs/Lab 2 - Build a copy pipeline.pdf and b/Labs/Lab 2 - Build a copy pipeline.pdf differ diff --git a/Labs/Lab 3 - Create a reusable pipeline.pdf b/Labs/Lab 3 - Create a reusable pipeline.pdf index 1637fa0..c40aad5 100644 Binary files a/Labs/Lab 3 - Create a reusable pipeline.pdf and b/Labs/Lab 3 - Create a reusable pipeline.pdf differ diff --git a/Labs/Lab 4 - Author a data flow.pdf b/Labs/Lab 4 - Author a data flow.pdf index 0ad1139..a018a21 100644 Binary files a/Labs/Lab 4 - Author a data flow.pdf and b/Labs/Lab 4 - Author a data flow.pdf differ diff --git a/Labs/Lab 5 - Integrate on-premises data.pdf b/Labs/Lab 5 - Integrate on-premises data.pdf new file mode 100644 index 0000000..41a46a5 Binary files /dev/null and b/Labs/Lab 5 - Integrate on-premises data.pdf differ diff --git a/Labs/Lab 5 - Monitor factory activity.pdf b/Labs/Lab 5 - Monitor factory activity.pdf deleted file mode 100644 index 93fd6e9..0000000 Binary files a/Labs/Lab 5 - Monitor factory activity.pdf and /dev/null differ diff --git a/Labs/Lab 6 - Explore Synapse pipelines.pdf b/Labs/Lab 6 - Explore Synapse pipelines.pdf deleted file mode 100644 index 22a6527..0000000 Binary files a/Labs/Lab 6 - Explore Synapse pipelines.pdf and /dev/null differ diff --git a/Labs/Lab 6 - Monitor factory activity.pdf b/Labs/Lab 6 - Monitor factory activity.pdf new file mode 100644 index 0000000..7ed5e20 Binary files /dev/null and b/Labs/Lab 6 - Monitor factory activity.pdf differ diff --git a/Labs/Lab 7 - Mini-project.pdf b/Labs/Lab 7 - Mini-project.pdf deleted file mode 100644 index d8190d9..0000000 Binary files a/Labs/Lab 7 - Mini-project.pdf and /dev/null differ diff --git a/Labs/TableCatalog.json b/Labs/TableCatalog.json deleted file mode 100644 index 8a66497..0000000 --- a/Labs/TableCatalog.json +++ /dev/null @@ -1,1082 +0,0 @@ -[ - { - "entityName": "Product", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 2 - }, - "sink": { - "name": "Name", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 3 - }, - "sink": { - "name": "ProductNumber", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 4 - }, - "sink": { - "name": "MakeFlag", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 5 - }, - "sink": { - "name": "FinishedGoodsFlag", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 6 - }, - "sink": { - "name": "Color", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 7 - }, - "sink": { - "name": "SafetyStockLevel", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 8 - }, - "sink": { - "name": "ReorderPoint", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 9 - }, - "sink": { - "name": "StandardCost", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 10 - }, - "sink": { - "name": "ListPrice", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 11 - }, - "sink": { - "name": "Size", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 12 - }, - "sink": { - "name": "SizeUnitMeasureCode", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 13 - }, - "sink": { - "name": "WeightUnitMeasureCode", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 14 - }, - "sink": { - "name": "Weight", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 15 - }, - "sink": { - "name": "DaysToManufacture", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 16 - }, - "sink": { - "name": "ProductLine", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 17 - }, - "sink": { - "name": "Class", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 18 - }, - "sink": { - "name": "Style", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 19 - }, - "sink": { - "name": "ProductSubcategoryID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 20 - }, - "sink": { - "name": "ProductModelID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 21 - }, - "sink": { - "name": "SellStartDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 22 - }, - "sink": { - "name": "SellEndDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 23 - }, - "sink": { - "name": "DiscontinuedDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Guid", - "ordinal": 24 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 25 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductCategory", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductCategoryID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 2 - }, - "sink": { - "name": "Name", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Guid", - "ordinal": 3 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 4 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductCostHistory", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 2 - }, - "sink": { - "name": "StartDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 3 - }, - "sink": { - "name": "EndDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 4 - }, - "sink": { - "name": "StandardCost", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 5 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductDescription", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductDescriptionID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 2 - }, - "sink": { - "name": "Description", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Guid", - "ordinal": 3 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 4 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductDocument", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 2 - }, - "sink": { - "name": "DocumentNode", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 3 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductInventory", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 2 - }, - "sink": { - "name": "LocationID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 3 - }, - "sink": { - "name": "Shelf", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 4 - }, - "sink": { - "name": "Bin", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 5 - }, - "sink": { - "name": "Quantity", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 6 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 7 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductListPriceHistory", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 2 - }, - "sink": { - "name": "StartDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 3 - }, - "sink": { - "name": "EndDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 4 - }, - "sink": { - "name": "ListPrice", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 5 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "ProductSubcategory", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "ProductSubcategoryID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 2 - }, - "sink": { - "name": "ProductCategoryID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 3 - }, - "sink": { - "name": "Name", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Guid", - "ordinal": 4 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 5 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "SalesOrderDetail", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "SalesOrderID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 2 - }, - "sink": { - "name": "SalesOrderDetailID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 3 - }, - "sink": { - "name": "CarrierTrackingNumber", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 4 - }, - "sink": { - "name": "OrderQty", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 5 - }, - "sink": { - "name": "ProductID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 6 - }, - "sink": { - "name": "SpecialOfferID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 7 - }, - "sink": { - "name": "UnitPrice", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 8 - }, - "sink": { - "name": "UnitPriceDiscount", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 9 - }, - "sink": { - "name": "LineTotal", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Guid", - "ordinal": 10 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 11 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - { - "entityName": "SalesOrderHeader", - "translator": { - "type": "TabularTranslator", - "mappings": [ - { - "source": { - "type": "Int32", - "ordinal": 1 - }, - "sink": { - "name": "SalesOrderID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 2 - }, - "sink": { - "name": "RevisionNumber", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 3 - }, - "sink": { - "name": "OrderDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 4 - }, - "sink": { - "name": "DueDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 5 - }, - "sink": { - "name": "ShipDate", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 6 - }, - "sink": { - "name": "Status", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 7 - }, - "sink": { - "name": "OnlineOrderFlag", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 8 - }, - "sink": { - "name": "SalesOrderNumber", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 9 - }, - "sink": { - "name": "PurchaseOrderNumber", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 10 - }, - "sink": { - "name": "AccountNumber", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 11 - }, - "sink": { - "name": "CustomerID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 12 - }, - "sink": { - "name": "SalesPersonID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 13 - }, - "sink": { - "name": "TerritoryID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 14 - }, - "sink": { - "name": "BillToAddressID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 15 - }, - "sink": { - "name": "ShipToAddressID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 16 - }, - "sink": { - "name": "ShipMethodID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 17 - }, - "sink": { - "name": "CreditCardID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 18 - }, - "sink": { - "name": "CreditCardApprovalCode", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Int32", - "ordinal": 19 - }, - "sink": { - "name": "CurrencyRateID", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 20 - }, - "sink": { - "name": "SubTotal", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 21 - }, - "sink": { - "name": "TaxAmt", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 22 - }, - "sink": { - "name": "Freight", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Decimal", - "ordinal": 23 - }, - "sink": { - "name": "TotalDue", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "String", - "ordinal": 24 - }, - "sink": { - "name": "Comment", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "Guid", - "ordinal": 25 - }, - "sink": { - "name": "rowguid", - "physicalType": "UTF8" - } - }, - { - "source": { - "type": "DateTime", - "ordinal": 26 - }, - "sink": { - "name": "ModifiedDate", - "physicalType": "UTF8" - } - } - ], - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - } -] \ No newline at end of file diff --git a/Labs/adf/dataflow/UpdateProductDimension.json b/Labs/adf/dataflow/UpdateProductDimension.json new file mode 100644 index 0000000..25ca4fc --- /dev/null +++ b/Labs/adf/dataflow/UpdateProductDimension.json @@ -0,0 +1,175 @@ +{ + "name": "UpdateProductDimension", + "properties": { + "folder": { + "name": "Lab4" + }, + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "Product" + }, + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "ProductSubcategory" + }, + { + "dataset": { + "referenceName": "ADLS_TSV_AdventureWorks", + "type": "DatasetReference" + }, + "name": "ProductCategory" + } + ], + "sinks": [ + { + "linkedService": { + "referenceName": "AzureDataLakeStorage1", + "type": "LinkedServiceReference" + }, + "name": "WriteToDataLake", + "description": "Write results to data lake" + } + ], + "transformations": [ + { + "name": "SelectProductColumns" + }, + { + "name": "SelectProductSubcategoryColumns" + }, + { + "name": "SelectProductCategoryColumns" + }, + { + "name": "LookupProductCategory", + "description": "Lookup product category name" + }, + { + "name": "LookupProductSubcategory" + }, + { + "name": "RemoveDuplicateColumns" + } + ], + "scriptLines": [ + "source(output(", + " ProductId as integer,", + " Product as string,", + " {_col2_} as string,", + " {_col3_} as boolean,", + " {_col4_} as boolean,", + " {_col5_} as string,", + " {_col6_} as short,", + " {_col7_} as short,", + " {_col8_} as double,", + " {_col9_} as double,", + " {_col10_} as string,", + " {_col11_} as string,", + " {_col12_} as string,", + " {_col13_} as double,", + " {_col14_} as short,", + " {_col15_} as string,", + " {_col16_} as string,", + " {_col17_} as string,", + " SubcategoryId as integer,", + " {_col19_} as short,", + " {_col20_} as timestamp,", + " {_col21_} as timestamp,", + " {_col22_} as string,", + " {_col23_} as string,", + " {_col24_} as string", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> Product", + "source(output(", + " SubcategoryId as integer,", + " CategoryId as short,", + " Subcategory as integer,", + " {_col3_} as string,", + " {_col4_} as timestamp", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> ProductSubcategory", + "source(output(", + " CategoryId as integer,", + " Category as string,", + " {_col2_} as string,", + " {_col3_} as timestamp", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " ignoreNoFilesFound: false) ~> ProductCategory", + "Product select(mapColumn(", + " ProductId,", + " Product,", + " SubcategoryId", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectProductColumns", + "ProductSubcategory select(mapColumn(", + " SubcategoryId,", + " CategoryId,", + " Subcategory", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectProductSubcategoryColumns", + "ProductCategory select(mapColumn(", + " CategoryId,", + " Category", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns", + "SelectProductSubcategoryColumns, SelectProductCategoryColumns lookup(SelectProductSubcategoryColumns@CategoryId == SelectProductCategoryColumns@CategoryId,", + " multiple: false,", + " pickup: 'any',", + " broadcast: 'auto')~> LookupProductCategory", + "SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectProductSubcategoryColumns@SubcategoryId,", + " multiple: false,", + " pickup: 'any',", + " broadcast: 'auto')~> LookupProductSubcategory", + "LookupProductSubcategory select(mapColumn(", + " ProductId,", + " Product,", + " SubcategoryId = SelectProductColumns@SubcategoryId,", + " SubcategoryId = SelectProductSubcategoryColumns@SubcategoryId,", + " CategoryId = SelectProductSubcategoryColumns@CategoryId,", + " Subcategory,", + " CategoryId = SelectProductCategoryColumns@CategoryId,", + " Category", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns", + "RemoveDuplicateColumns sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'delta',", + " fileSystem: 'lakeroot',", + " folderPath: 'Prepared/DimProduct',", + " overwrite:true,", + " mergeSchema: false,", + " autoCompact: false,", + " optimizedWrite: false,", + " vacuum: 0,", + " deletable:false,", + " insertable:true,", + " updateable:false,", + " upsertable:false,", + " umask: 0022,", + " preCommands: [],", + " postCommands: [],", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> WriteToDataLake" + ] + } + } +} \ No newline at end of file diff --git a/Labs/adf/dataset/ADLS_BIN_AWProduct.json b/Labs/adf/dataset/ADLS_BIN_AWProduct.json new file mode 100644 index 0000000..8dddcd6 --- /dev/null +++ b/Labs/adf/dataset/ADLS_BIN_AWProduct.json @@ -0,0 +1,23 @@ +{ + "name": "ADLS_BIN_AWProduct", + "properties": { + "linkedServiceName": { + "referenceName": "AzureDataLakeStorage1", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lab2" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "Product.csv", + "folderPath": "Raw", + "fileSystem": "lakeroot" + } + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Labs/adf/dataset/ADLS_BIN_AdventureWorks.json b/Labs/adf/dataset/ADLS_BIN_AdventureWorks.json new file mode 100644 index 0000000..3be595e --- /dev/null +++ b/Labs/adf/dataset/ADLS_BIN_AdventureWorks.json @@ -0,0 +1,31 @@ +{ + "name": "ADLS_BIN_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "AzureDataLakeStorage1", + "type": "LinkedServiceReference" + }, + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Lab3" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@{dataset().Entity}.tsv", + "type": "Expression" + }, + "folderPath": "Raw", + "fileSystem": "lakeroot" + } + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Labs/adf/dataset/ADLS_MyLocalFile.json b/Labs/adf/dataset/ADLS_MyLocalFile.json new file mode 100644 index 0000000..098e9e7 --- /dev/null +++ b/Labs/adf/dataset/ADLS_MyLocalFile.json @@ -0,0 +1,22 @@ +{ + "name": "ADLS_MyLocalFile", + "properties": { + "linkedServiceName": { + "referenceName": "AzureDataLakeStorage1", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lab5" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "mylocalfile.txt", + "folderPath": "Raw", + "fileSystem": "lakeroot" + } + } + } +} \ No newline at end of file diff --git a/Labs/adf/dataset/ADLS_TSV_AdventureWorks.json b/Labs/adf/dataset/ADLS_TSV_AdventureWorks.json new file mode 100644 index 0000000..9029a54 --- /dev/null +++ b/Labs/adf/dataset/ADLS_TSV_AdventureWorks.json @@ -0,0 +1,35 @@ +{ + "name": "ADLS_TSV_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "AzureDataLakeStorage1", + "type": "LinkedServiceReference" + }, + "parameters": { + "FileName": { + "type": "string" + } + }, + "folder": { + "name": "Lab4" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().FileName", + "type": "Expression" + }, + "folderPath": "Raw", + "fileSystem": "lakeroot" + }, + "columnDelimiter": "\t", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Labs/adf/dataset/FS_MyFiles.json b/Labs/adf/dataset/FS_MyFiles.json new file mode 100644 index 0000000..5444fde --- /dev/null +++ b/Labs/adf/dataset/FS_MyFiles.json @@ -0,0 +1,21 @@ +{ + "name": "FS_MyFiles", + "properties": { + "linkedServiceName": { + "referenceName": "FS_MyLaptop", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lab5" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": "data-file.txt", + "folderPath": "tmp" + } + } + } +} \ No newline at end of file diff --git a/Labs/adf/dataset/HTTP_BIN_AWProduct.json b/Labs/adf/dataset/HTTP_BIN_AWProduct.json new file mode 100644 index 0000000..12573bf --- /dev/null +++ b/Labs/adf/dataset/HTTP_BIN_AWProduct.json @@ -0,0 +1,20 @@ +{ + "name": "HTTP_BIN_AWProduct", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWProduct", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lab2" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "HttpServerLocation" + } + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Labs/adf/dataset/HTTP_BIN_AdventureWorks.json b/Labs/adf/dataset/HTTP_BIN_AdventureWorks.json new file mode 100644 index 0000000..6c86213 --- /dev/null +++ b/Labs/adf/dataset/HTTP_BIN_AdventureWorks.json @@ -0,0 +1,29 @@ +{ + "name": "HTTP_BIN_AdventureWorks", + "properties": { + "linkedServiceName": { + "referenceName": "HTTP_AWGitHub", + "type": "LinkedServiceReference" + }, + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Lab3" + }, + "annotations": [], + "type": "Binary", + "typeProperties": { + "location": { + "type": "HttpServerLocation", + "relativeUrl": { + "value": "@{dataset().Entity}.csv", + "type": "Expression" + } + } + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Labs/adf/factory/adflabs2023-adf.json b/Labs/adf/factory/adflabs2023-adf.json new file mode 100644 index 0000000..58f6ba6 --- /dev/null +++ b/Labs/adf/factory/adflabs2023-adf.json @@ -0,0 +1,9 @@ +{ + "name": "adflabs2023-adf", + "location": "uksouth", + "identity": { + "type": "SystemAssigned", + "principalId": "b9cd3fd6-9668-41f7-ae62-413b409b5f58", + "tenantId": "2cdc9c7e-334f-4f88-9de8-a844761a49b6" + } +} \ No newline at end of file diff --git a/Labs/adf/integrationRuntime/SHIR.json b/Labs/adf/integrationRuntime/SHIR.json new file mode 100644 index 0000000..43fbfa8 --- /dev/null +++ b/Labs/adf/integrationRuntime/SHIR.json @@ -0,0 +1,6 @@ +{ + "name": "SHIR", + "properties": { + "type": "SelfHosted" + } +} \ No newline at end of file diff --git a/Labs/adf/linkedService/AzureDataLakeStorage1.json b/Labs/adf/linkedService/AzureDataLakeStorage1.json new file mode 100644 index 0000000..843a065 --- /dev/null +++ b/Labs/adf/linkedService/AzureDataLakeStorage1.json @@ -0,0 +1,11 @@ +{ + "name": "AzureDataLakeStorage1", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "https://adflabs2023sa.dfs.core.windows.net/" + } + } +} \ No newline at end of file diff --git a/Labs/adf/linkedService/FS_MyLaptop.json b/Labs/adf/linkedService/FS_MyLaptop.json new file mode 100644 index 0000000..e0c8237 --- /dev/null +++ b/Labs/adf/linkedService/FS_MyLaptop.json @@ -0,0 +1,17 @@ +{ + "name": "FS_MyLaptop", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "FileServer", + "typeProperties": { + "host": "C:\\", + "userId": "richard", + "encryptedCredential": "eyJDcmVkZW50aWFsSWQiOiIzZDk3NmY1My0zNGYzLTQ1NWUtYjE0MS01ZGMyYmI0MmY2YmQiLCJWZXJzaW9uIjoiMi4wIiwiQ2xhc3NUeXBlIjoiTWljcm9zb2Z0LkRhdGFQcm94eS5Db3JlLkludGVyU2VydmljZURhdGFDb250cmFjdC5DcmVkZW50aWFsU1UwNkNZMTQifQ==" + }, + "connectVia": { + "referenceName": "SHIR", + "type": "IntegrationRuntimeReference" + } + } +} \ No newline at end of file diff --git a/Labs/adf/linkedService/HTTP_AWGitHub.json b/Labs/adf/linkedService/HTTP_AWGitHub.json new file mode 100644 index 0000000..eede41b --- /dev/null +++ b/Labs/adf/linkedService/HTTP_AWGitHub.json @@ -0,0 +1,13 @@ +{ + "name": "HTTP_AWGitHub", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "HttpServer", + "typeProperties": { + "url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/", + "enableServerCertificateValidation": true, + "authenticationType": "Anonymous" + } + } +} \ No newline at end of file diff --git a/Labs/adf/linkedService/HTTP_AWProduct.json b/Labs/adf/linkedService/HTTP_AWProduct.json new file mode 100644 index 0000000..d84c99d --- /dev/null +++ b/Labs/adf/linkedService/HTTP_AWProduct.json @@ -0,0 +1,13 @@ +{ + "name": "HTTP_AWProduct", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "HttpServer", + "typeProperties": { + "url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv", + "enableServerCertificateValidation": true, + "authenticationType": "Anonymous" + } + } +} \ No newline at end of file diff --git a/Labs/adf/pipeline/BuildDimProduct.json b/Labs/adf/pipeline/BuildDimProduct.json new file mode 100644 index 0000000..36577a9 --- /dev/null +++ b/Labs/adf/pipeline/BuildDimProduct.json @@ -0,0 +1,48 @@ +{ + "name": "BuildDimProduct", + "properties": { + "activities": [ + { + "name": "Run UpdateProductDimension", + "type": "ExecuteDataFlow", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataflow": { + "referenceName": "UpdateProductDimension", + "type": "DataFlowReference", + "datasetParameters": { + "Product": { + "FileName": "Product.tsv" + }, + "ProductSubcategory": { + "FileName": "ProductSubcategory.tsv" + }, + "ProductCategory": { + "FileName": "ProductCategory.tsv" + } + } + }, + "compute": { + "coreCount": 8, + "computeType": "General" + }, + "traceLevel": "Fine" + } + } + ], + "folder": { + "name": "Lab4" + }, + "annotations": [], + "lastPublishTime": "2023-05-28T17:33:03Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Labs/adf/pipeline/CopyAWEntities.json b/Labs/adf/pipeline/CopyAWEntities.json new file mode 100644 index 0000000..b92ce21 --- /dev/null +++ b/Labs/adf/pipeline/CopyAWEntities.json @@ -0,0 +1,56 @@ +{ + "name": "CopyAWEntities", + "properties": { + "activities": [ + { + "name": "Copy each AW entity", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@variables('Entities')", + "type": "Expression" + }, + "activities": [ + { + "name": "Execute CopyAWEntity", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "CopyAWEntity", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "Entity": { + "value": "@item()", + "type": "Expression" + } + } + } + } + ] + } + } + ], + "variables": { + "Entities": { + "type": "Array", + "defaultValue": [ + "Product", + "ProductSubcategory", + "ProductCategory" + ] + } + }, + "folder": { + "name": "Lab3" + }, + "annotations": [], + "lastPublishTime": "2023-05-28T14:23:19Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Labs/adf/pipeline/CopyAWEntity.json b/Labs/adf/pipeline/CopyAWEntity.json new file mode 100644 index 0000000..65d337f --- /dev/null +++ b/Labs/adf/pipeline/CopyAWEntity.json @@ -0,0 +1,74 @@ +{ + "name": "CopyAWEntity", + "properties": { + "activities": [ + { + "name": "Copy AW entity", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "HTTP_BIN_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "Entity": { + "value": "@pipeline().parameters.Entity", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "ADLS_BIN_AdventureWorks", + "type": "DatasetReference", + "parameters": { + "Entity": { + "value": "@pipeline().parameters.Entity", + "type": "Expression" + } + } + } + ] + } + ], + "parameters": { + "Entity": { + "type": "string" + } + }, + "folder": { + "name": "Lab3" + }, + "annotations": [], + "lastPublishTime": "2023-05-28T14:05:42Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Labs/adf/pipeline/CopyAWProduct.json b/Labs/adf/pipeline/CopyAWProduct.json new file mode 100644 index 0000000..a862391 --- /dev/null +++ b/Labs/adf/pipeline/CopyAWProduct.json @@ -0,0 +1,57 @@ +{ + "name": "CopyAWProduct", + "properties": { + "activities": [ + { + "name": "Copy AW Products to data lake", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "HttpReadSettings", + "requestMethod": "GET" + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "HTTP_BIN_AWProduct", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ADLS_BIN_AWProduct", + "type": "DatasetReference" + } + ] + } + ], + "folder": { + "name": "Lab2" + }, + "annotations": [], + "lastPublishTime": "2023-05-27T20:00:12Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Labs/adf/pipeline/CopyLocalFile.json b/Labs/adf/pipeline/CopyLocalFile.json new file mode 100644 index 0000000..3cb9633 --- /dev/null +++ b/Labs/adf/pipeline/CopyLocalFile.json @@ -0,0 +1,57 @@ +{ + "name": "CopyLocalFile", + "properties": { + "activities": [ + { + "name": "Copy AW Products to data lake", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "BinarySource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "BinaryReadSettings" + } + }, + "sink": { + "type": "BinarySink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + } + }, + "enableStaging": false + }, + "inputs": [ + { + "referenceName": "FS_MyFiles", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "ADLS_MyLocalFile", + "type": "DatasetReference" + } + ] + } + ], + "folder": { + "name": "Lab5" + }, + "annotations": [], + "lastPublishTime": "2023-05-27T20:00:12Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Labs/adf/publish_config.json b/Labs/adf/publish_config.json new file mode 100644 index 0000000..80b9cca --- /dev/null +++ b/Labs/adf/publish_config.json @@ -0,0 +1 @@ +{"publishBranch":"adf_publish","enableGitComment":true} \ No newline at end of file