diff --git a/Code/DataFactory/credential/TrainingCredential01.json b/Code/DataFactory/credential/TrainingCredential01.json new file mode 100644 index 0000000..855050a --- /dev/null +++ b/Code/DataFactory/credential/TrainingCredential01.json @@ -0,0 +1,9 @@ +{ + "name": "TrainingCredential01", + "properties": { + "type": "ManagedIdentity", + "typeProperties": { + "resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataflow/MappingOrderAggregation.json b/Code/DataFactory/dataflow/MappingOrderAggregation.json new file mode 100644 index 0000000..a459096 --- /dev/null +++ b/Code/DataFactory/dataflow/MappingOrderAggregation.json @@ -0,0 +1,42 @@ +{ + "name": "MappingOrderAggregation", + "properties": { + "type": "MappingDataFlow", + "typeProperties": { + "sources": [ + { + "dataset": { + "referenceName": "LakeFileOrderHeaderParquet", + "type": "DatasetReference" + }, + "name": "OrderHeader" + }, + { + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + }, + "name": "OrderLineDetails" + } + ], + "sinks": [ + { + "dataset": { + "referenceName": "TableOrderSummary", + "type": "DatasetReference" + }, + "name": "OrderSummary" + } + ], + "transformations": [ + { + "name": "JoinHeaderToLineDetails" + }, + { + "name": "OrderLineCount" + } + ], + "script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataflow/WranglingOrderAggregation.json b/Code/DataFactory/dataflow/WranglingOrderAggregation.json new file mode 100644 index 0000000..fae0217 --- /dev/null +++ b/Code/DataFactory/dataflow/WranglingOrderAggregation.json @@ -0,0 +1,19 @@ +{ + "name": "WranglingOrderAggregation", + "properties": { + "type": "WranglingDataFlow", + "typeProperties": { + "sources": [ + { + "name": "LakeFileOrderDetailLinesParquet", + "script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet", + "dataset": { + "referenceName": "LakeFileOrderDetailLinesParquet", + "type": "DatasetReference" + } + } + ], + "script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/AnyDatabaseTable.json b/Code/DataFactory/dataset/AnyDatabaseTable.json new file mode 100644 index 0000000..57f89e5 --- /dev/null +++ b/Code/DataFactory/dataset/AnyDatabaseTable.json @@ -0,0 +1,43 @@ +{ + "name": "AnyDatabaseTable", + "properties": { + "linkedServiceName": { + "referenceName": "AnyDatabaseConnection", + "type": "LinkedServiceReference", + "parameters": { + "DBConnectionSecret": { + "value": "@dataset().LinkedServiceConnectionSecret", + "type": "Expression" + } + } + }, + "parameters": { + "LinkedServiceConnectionSecret": { + "type": "string" + }, + "SchemaName": { + "type": "string" + }, + "TableName": { + "type": "string" + } + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": { + "schema": { + "value": "@dataset().SchemaName", + "type": "Expression" + }, + "table": { + "value": "@dataset().TableName", + "type": "Expression" + } + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/GetSetMetadata.json b/Code/DataFactory/dataset/GetSetMetadata.json new file mode 100644 index 0000000..4d21635 --- /dev/null +++ b/Code/DataFactory/dataset/GetSetMetadata.json @@ -0,0 +1,16 @@ +{ + "name": "GetSetMetadata", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json b/Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json new file mode 100644 index 0000000..0cbd145 --- /dev/null +++ b/Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json @@ -0,0 +1,68 @@ +{ + "name": "LakeFileOrderDetailLinesParquet", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01noneKV", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "OrderDetailLines.parquet", + "folderPath": "Raw", + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [ + { + "name": "SalesOrderID", + "type": "INT32" + }, + { + "name": "SalesOrderDetailID", + "type": "INT32" + }, + { + "name": "OrderQty", + "type": "INT32" + }, + { + "name": "ProductID", + "type": "INT32" + }, + { + "name": "UnitPrice", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "UnitPriceDiscount", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "LineTotal", + "type": "DECIMAL", + "precision": 38, + "scale": 6 + }, + { + "name": "rowguid", + "type": "UTF8" + }, + { + "name": "ModifiedDate", + "type": "INT96" + } + ] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json b/Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json new file mode 100644 index 0000000..904aed8 --- /dev/null +++ b/Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json @@ -0,0 +1,122 @@ +{ + "name": "LakeFileOrderHeaderParquet", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01noneKV", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "OrderHeader.parquet", + "folderPath": "Raw", + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [ + { + "name": "SalesOrderID", + "type": "INT32" + }, + { + "name": "RevisionNumber", + "type": "INT32" + }, + { + "name": "OrderDate", + "type": "INT96" + }, + { + "name": "DueDate", + "type": "INT96" + }, + { + "name": "ShipDate", + "type": "INT96" + }, + { + "name": "Status", + "type": "INT32" + }, + { + "name": "OnlineOrderFlag", + "type": "BOOLEAN" + }, + { + "name": "SalesOrderNumber", + "type": "UTF8" + }, + { + "name": "PurchaseOrderNumber", + "type": "UTF8" + }, + { + "name": "AccountNumber", + "type": "UTF8" + }, + { + "name": "CustomerID", + "type": "INT32" + }, + { + "name": "ShipToAddressID", + "type": "INT32" + }, + { + "name": "BillToAddressID", + "type": "INT32" + }, + { + "name": "ShipMethod", + "type": "UTF8" + }, + { + "name": "CreditCardApprovalCode", + "type": "UTF8" + }, + { + "name": "SubTotal", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "TaxAmt", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "Freight", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "TotalDue", + "type": "DECIMAL", + "precision": 19, + "scale": 4 + }, + { + "name": "Comment", + "type": "UTF8" + }, + { + "name": "rowguid", + "type": "UTF8" + }, + { + "name": "ModifiedDate", + "type": "INT96" + } + ] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LakeFileParquet.json b/Code/DataFactory/dataset/LakeFileParquet.json new file mode 100644 index 0000000..e7b3ad3 --- /dev/null +++ b/Code/DataFactory/dataset/LakeFileParquet.json @@ -0,0 +1,39 @@ +{ + "name": "LakeFileParquet", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "Parquet", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "compressionCodec": "snappy" + }, + "schema": [] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LakeFilePersonCSV.json b/Code/DataFactory/dataset/LakeFilePersonCSV.json new file mode 100644 index 0000000..83f29db --- /dev/null +++ b/Code/DataFactory/dataset/LakeFilePersonCSV.json @@ -0,0 +1,37 @@ +{ + "name": "LakeFilePersonCSV", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": "Person.csv", + "folderPath": "Landing", + "fileSystem": "datawarehouse" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [ + { + "type": "String" + }, + { + "type": "String" + }, + { + "type": "String" + } + ] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LakeFiles.json b/Code/DataFactory/dataset/LakeFiles.json new file mode 100644 index 0000000..79d1e7e --- /dev/null +++ b/Code/DataFactory/dataset/LakeFiles.json @@ -0,0 +1,41 @@ +{ + "name": "LakeFiles", + "properties": { + "linkedServiceName": { + "referenceName": "traininglake01", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Lake" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + }, + "fileSystem": "datawarehouse" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LaptopFilePersonCSV.json b/Code/DataFactory/dataset/LaptopFilePersonCSV.json new file mode 100644 index 0000000..b9fbd58 --- /dev/null +++ b/Code/DataFactory/dataset/LaptopFilePersonCSV.json @@ -0,0 +1,36 @@ +{ + "name": "LaptopFilePersonCSV", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": "Person.csv", + "folderPath": "ForUpload/People" + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [ + { + "type": "String" + }, + { + "type": "String" + }, + { + "type": "String" + } + ] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LaptopFolders.json b/Code/DataFactory/dataset/LaptopFolders.json new file mode 100644 index 0000000..595d0ef --- /dev/null +++ b/Code/DataFactory/dataset/LaptopFolders.json @@ -0,0 +1,33 @@ +{ + "name": "LaptopFolders", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/LaptopsFiles.json b/Code/DataFactory/dataset/LaptopsFiles.json new file mode 100644 index 0000000..a4c42a6 --- /dev/null +++ b/Code/DataFactory/dataset/LaptopsFiles.json @@ -0,0 +1,40 @@ +{ + "name": "LaptopsFiles", + "properties": { + "linkedServiceName": { + "referenceName": "LaptopFiles", + "type": "LinkedServiceReference" + }, + "parameters": { + "Directory": { + "type": "string" + }, + "File": { + "type": "string" + } + }, + "folder": { + "name": "Laptop" + }, + "annotations": [], + "type": "DelimitedText", + "typeProperties": { + "location": { + "type": "FileServerLocation", + "fileName": { + "value": "@dataset().File", + "type": "Expression" + }, + "folderPath": { + "value": "@dataset().Directory", + "type": "Expression" + } + }, + "columnDelimiter": ",", + "escapeChar": "\\", + "quoteChar": "\"" + }, + "schema": [] + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/dataset/TableOrderSummary.json b/Code/DataFactory/dataset/TableOrderSummary.json new file mode 100644 index 0000000..65aebcf --- /dev/null +++ b/Code/DataFactory/dataset/TableOrderSummary.json @@ -0,0 +1,30 @@ +{ + "name": "TableOrderSummary", + "properties": { + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + }, + "folder": { + "name": "SQLDB" + }, + "annotations": [], + "type": "AzureSqlTable", + "schema": [ + { + "name": "SalesOrderNumber", + "type": "varchar" + }, + { + "name": "RecordCount", + "type": "int", + "precision": 10 + } + ], + "typeProperties": { + "schema": "dbo", + "table": "OrderSummary" + } + }, + "type": "Microsoft.DataFactory/factories/datasets" +} \ No newline at end of file diff --git a/Code/DataFactory/factory/TrainingFactoryDev.json b/Code/DataFactory/factory/TrainingFactoryDev.json new file mode 100644 index 0000000..195503a --- /dev/null +++ b/Code/DataFactory/factory/TrainingFactoryDev.json @@ -0,0 +1,15 @@ +{ + "name": "TrainingFactoryDev", + "location": "uksouth", + "identity": { + "type": "SystemAssigned,UserAssigned", + "principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f", + "tenantId": "929039b9-c19d-425e-993b-1e3fda155876", + "userAssignedIdentities": { + "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": { + "clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85", + "principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/integrationRuntime/ForDataFlowDemos.json b/Code/DataFactory/integrationRuntime/ForDataFlowDemos.json new file mode 100644 index 0000000..b974304 --- /dev/null +++ b/Code/DataFactory/integrationRuntime/ForDataFlowDemos.json @@ -0,0 +1,16 @@ +{ + "name": "ForDataFlowDemos", + "properties": { + "type": "Managed", + "typeProperties": { + "computeProperties": { + "location": "UK South", + "dataFlowProperties": { + "computeType": "General", + "coreCount": 8, + "timeToLive": 240 + } + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/integrationRuntime/PaulsLaptopIR.json b/Code/DataFactory/integrationRuntime/PaulsLaptopIR.json new file mode 100644 index 0000000..81a2992 --- /dev/null +++ b/Code/DataFactory/integrationRuntime/PaulsLaptopIR.json @@ -0,0 +1,12 @@ +{ + "name": "PaulsLaptopIR", + "properties": { + "type": "SelfHosted", + "typeProperties": { + "linkedInfo": { + "resourceId": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WorkLaptop", + "authorizationType": "Rbac" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/integrationRuntime/VNetEnabledIR.json b/Code/DataFactory/integrationRuntime/VNetEnabledIR.json new file mode 100644 index 0000000..220b407 --- /dev/null +++ b/Code/DataFactory/integrationRuntime/VNetEnabledIR.json @@ -0,0 +1,21 @@ +{ + "name": "VNetEnabledIR", + "properties": { + "type": "Managed", + "typeProperties": { + "computeProperties": { + "location": "UK South", + "dataFlowProperties": { + "computeType": "General", + "coreCount": 8, + "timeToLive": 0, + "cleanup": true + } + } + }, + "managedVirtualNetwork": { + "type": "ManagedVirtualNetworkReference", + "referenceName": "default" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/AnyDatabaseConnection.json b/Code/DataFactory/linkedService/AnyDatabaseConnection.json new file mode 100644 index 0000000..d97b8f0 --- /dev/null +++ b/Code/DataFactory/linkedService/AnyDatabaseConnection.json @@ -0,0 +1,26 @@ +{ + "name": "AnyDatabaseConnection", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "parameters": { + "DBConnectionSecret": { + "type": "string" + } + }, + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": { + "value": "@linkedService().DBConnectionSecret", + "type": "Expression" + } + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/BatchForTraining01.json b/Code/DataFactory/linkedService/BatchForTraining01.json new file mode 100644 index 0000000..a7231c0 --- /dev/null +++ b/Code/DataFactory/linkedService/BatchForTraining01.json @@ -0,0 +1,17 @@ +{ + "name": "BatchForTraining01", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBatch", + "typeProperties": { + "batchUri": "https://batchfortraining01.uksouth.batch.azure.com", + "poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654", + "accountName": "batchfortraining01", + "linkedServiceName": { + "referenceName": "TrainingStore01", + "type": "LinkedServiceReference" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/LaptopFiles.json b/Code/DataFactory/linkedService/LaptopFiles.json new file mode 100644 index 0000000..7ab03b8 --- /dev/null +++ b/Code/DataFactory/linkedService/LaptopFiles.json @@ -0,0 +1,24 @@ +{ + "name": "LaptopFiles", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "FileServer", + "typeProperties": { + "host": "C:\\ADFRoot\\", + "userId": "ADFIRUser", + "password": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "ADFIRUser" + } + }, + "connectVia": { + "referenceName": "PaulsLaptopIR", + "type": "IntegrationRuntimeReference" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/TrainingKeys01.json b/Code/DataFactory/linkedService/TrainingKeys01.json new file mode 100644 index 0000000..5af4588 --- /dev/null +++ b/Code/DataFactory/linkedService/TrainingKeys01.json @@ -0,0 +1,11 @@ +{ + "name": "TrainingKeys01", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "https://TrainingKeys01.vault.azure.net/" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/TrainingKeys01withUMI.json b/Code/DataFactory/linkedService/TrainingKeys01withUMI.json new file mode 100644 index 0000000..e403353 --- /dev/null +++ b/Code/DataFactory/linkedService/TrainingKeys01withUMI.json @@ -0,0 +1,15 @@ +{ + "name": "TrainingKeys01withUMI", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "https://TrainingKeys01.vault.azure.net/", + "credential": { + "referenceName": "TrainingCredential01", + "type": "CredentialReference" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/TrainingStore01.json b/Code/DataFactory/linkedService/TrainingStore01.json new file mode 100644 index 0000000..b299521 --- /dev/null +++ b/Code/DataFactory/linkedService/TrainingStore01.json @@ -0,0 +1,12 @@ +{ + "name": "TrainingStore01", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobStorage", + "typeProperties": { + "connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;", + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/trainingdb01.json b/Code/DataFactory/linkedService/trainingdb01.json new file mode 100644 index 0000000..a118e82 --- /dev/null +++ b/Code/DataFactory/linkedService/trainingdb01.json @@ -0,0 +1,18 @@ +{ + "name": "trainingdb01", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "ConnectionString-trainingdb01" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/traininglak01.json b/Code/DataFactory/linkedService/traininglak01.json new file mode 100644 index 0000000..083ff82 --- /dev/null +++ b/Code/DataFactory/linkedService/traininglak01.json @@ -0,0 +1,12 @@ +{ + "name": "traininglak01", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "https://traininglake01.dfs.core.windows.net", + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/traininglake01.json b/Code/DataFactory/linkedService/traininglake01.json new file mode 100644 index 0000000..14295af --- /dev/null +++ b/Code/DataFactory/linkedService/traininglake01.json @@ -0,0 +1,19 @@ +{ + "name": "traininglake01", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "https://traininglake01.dfs.core.windows.net", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "TrainingKeys01", + "type": "LinkedServiceReference" + }, + "secretName": "traininglake01" + } + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/linkedService/traininglake01noneKV.json b/Code/DataFactory/linkedService/traininglake01noneKV.json new file mode 100644 index 0000000..9795d8d --- /dev/null +++ b/Code/DataFactory/linkedService/traininglake01noneKV.json @@ -0,0 +1,12 @@ +{ + "name": "traininglake01noneKV", + "type": "Microsoft.DataFactory/factories/linkedservices", + "properties": { + "annotations": [], + "type": "AzureBlobFS", + "typeProperties": { + "url": "https://traininglake01.dfs.core.windows.net", + "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9" + } + } +} \ No newline at end of file diff --git a/Code/DataFactory/managedVirtualNetwork/default.json b/Code/DataFactory/managedVirtualNetwork/default.json new file mode 100644 index 0000000..6cbc29f --- /dev/null +++ b/Code/DataFactory/managedVirtualNetwork/default.json @@ -0,0 +1,4 @@ +{ + "name": "default", + "type": "Microsoft.DataFactory/factories/managedvirtualnetworks" +} \ No newline at end of file diff --git a/Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureKeyVault1.json b/Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureKeyVault1.json new file mode 100644 index 0000000..6b70a04 --- /dev/null +++ b/Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureKeyVault1.json @@ -0,0 +1,10 @@ +{ + "name": "AzureKeyVault1", + "properties": { + "privateLinkResourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01", + "groupId": "vault", + "fqdns": [ + "trainingkeys01.vaultcore.azure.net" + ] + } +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/01 - Upload - Simple.json b/Code/DataFactory/pipeline/01 - Upload - Simple.json new file mode 100644 index 0000000..9fa1b4c --- /dev/null +++ b/Code/DataFactory/pipeline/01 - Upload - Simple.json @@ -0,0 +1,71 @@ +{ + "name": "01 - Upload - Simple", + "properties": { + "activities": [ + { + "name": "Upload Person", + "description": "hfdhgfdhfd", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopFilePersonCSV", + "type": "DatasetReference" + } + ], + "outputs": [ + { + "referenceName": "LakeFilePersonCSV", + "type": "DatasetReference" + } + ] + } + ], + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-29T13:50:28Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/02 - Upload - Copy Params.json b/Code/DataFactory/pipeline/02 - Upload - Copy Params.json new file mode 100644 index 0000000..1d00ef9 --- /dev/null +++ b/Code/DataFactory/pipeline/02 - Upload - Copy Params.json @@ -0,0 +1,78 @@ +{ + "name": "02 - Upload - Copy Params", + "properties": { + "activities": [ + { + "name": "Upload Any File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": "Person.csv" + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": "Person.csv" + } + } + ] + } + ], + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:46Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/03 - Upload - From Discovery.json b/Code/DataFactory/pipeline/03 - Upload - From Discovery.json new file mode 100644 index 0000000..f7ac2ca --- /dev/null +++ b/Code/DataFactory/pipeline/03 - Upload - From Discovery.json @@ -0,0 +1,137 @@ +{ + "name": "03 - Upload - From Discovery", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "GetMetadata", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "LaptopFolders", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People" + } + }, + "fieldList": [ + "childItems" + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + } + }, + { + "name": "Upload Files", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.childItems", + "type": "Expression" + }, + "activities": [ + { + "name": "Upload File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "ForUpload\\People", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": "Landing", + "File": { + "value": "@item().name", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:06:45Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/04 - Upload - From Metadata.json b/Code/DataFactory/pipeline/04 - Upload - From Metadata.json new file mode 100644 index 0000000..068db8c --- /dev/null +++ b/Code/DataFactory/pipeline/04 - Upload - From Metadata.json @@ -0,0 +1,266 @@ +{ + "name": "04 - Upload - From Metadata", + "properties": { + "activities": [ + { + "name": "Get File List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference" + }, + "firstRowOnly": false + } + }, + { + "name": "UploadFiles", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Log Upload Start", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get File List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Upload File", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "additionalColumns": [ + { + "name": "FilePath", + "value": "$$FILEPATH" + }, + { + "name": "DataFactoryName", + "value": { + "value": "@pipeline().DataFactory", + "type": "Expression" + } + }, + { + "name": "PipelineName", + "value": { + "value": "@pipeline().Pipeline", + "type": "Expression" + } + }, + { + "name": "UploadDateTime", + "value": { + "value": "@utcnow()", + "type": "Expression" + } + }, + { + "name": "RunId", + "value": { + "value": "@pipeline().RunId", + "type": "Expression" + } + } + ], + "storeSettings": { + "type": "FileServerReadSettings", + "recursive": true + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": { + "type": "DelimitedTextWriteSettings", + "quoteAllText": true, + "fileExtension": ".txt" + } + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "LaptopsFiles", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().SourceDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "LakeFiles", + "type": "DatasetReference", + "parameters": { + "Directory": { + "value": "@item().TargetDirectory", + "type": "Expression" + }, + "File": { + "value": "@item().FileName", + "type": "Expression" + } + } + } + ] + } + ] + } + }, + { + "name": "Log Upload Start", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Get File List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[dbo].[LogUploadStart]", + "storedProcedureParameters": { + "FileCount": { + "type": "Int32", + "value": { + "value": "@activity('Get File List').output.count", + "type": "Expression" + } + }, + "TriggerId": { + "type": "String", + "value": { + "value": "@pipeline().TriggerId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference" + } + } + }, + { + "name": "Log Upload Finish", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "UploadFiles", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[dbo].[LogUploadFinish]", + "storedProcedureParameters": { + "LogId": { + "value": { + "value": "@activity('Log Upload Start').output.firstRow.LogId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ], + "variables": { + "DemoVariable": { + "type": "String" + } + }, + "folder": { + "name": "Demo Pipelines/Data Uploads" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:14Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/1950 Activities.json b/Code/DataFactory/pipeline/1950 Activities.json new file mode 100644 index 0000000..4c00f16 --- /dev/null +++ b/Code/DataFactory/pipeline/1950 Activities.json @@ -0,0 +1,379 @@ +{ + "name": "1950 Activities", + "properties": { + "activities": [ + { + "name": "50", + "type": "ForEach", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Wait 1 Minute 1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 2", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 3", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 4", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 5", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 6", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 7", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 8", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 9", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 10", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 11", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 12", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 13", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 14", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 15", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 16", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 17", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 18", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 19", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 20", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 21", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 22", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 23", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 24", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 25", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 26", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 27", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 28", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 29", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 30", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 31", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 32", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 33", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 34", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 35", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 36", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 37", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 38", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + }, + { + "name": "Wait 1 Minute 39", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 600 + } + } + ] + } + } + ], + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [], + "lastPublishTime": "2022-02-03T14:39:03Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Custom Activity.json b/Code/DataFactory/pipeline/Custom Activity.json new file mode 100644 index 0000000..0bc5625 --- /dev/null +++ b/Code/DataFactory/pipeline/Custom Activity.json @@ -0,0 +1,42 @@ +{ + "name": "Custom Activity", + "properties": { + "activities": [ + { + "name": "HelloWorld", + "type": "Custom", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "command": "CustomActivity.exe", + "resourceLinkedService": { + "referenceName": "TrainingStore01", + "type": "LinkedServiceReference" + }, + "folderPath": "customactivities", + "referenceObjects": { + "linkedServices": [], + "datasets": [] + } + }, + "linkedServiceName": { + "referenceName": "BatchForTraining01", + "type": "LinkedServiceReference" + } + } + ], + "folder": { + "name": "Demo Pipelines/Misc" + }, + "annotations": [], + "lastPublishTime": "2022-02-03T14:39:04Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Lazy Replication.json b/Code/DataFactory/pipeline/Lazy Replication.json new file mode 100644 index 0000000..c0f4b40 --- /dev/null +++ b/Code/DataFactory/pipeline/Lazy Replication.json @@ -0,0 +1,163 @@ +{ + "name": "Lazy Replication", + "properties": { + "activities": [ + { + "name": "Get Table List", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderQuery": { + "value": "@pipeline().parameters.TableLookupQuery", + "type": "Expression" + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": "sys", + "TableName": "objects" + } + }, + "firstRowOnly": false + } + }, + { + "name": "Copy Tables", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Table List", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Table List').output.value", + "type": "Expression" + }, + "activities": [ + { + "name": "Copy Table", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "sink": { + "type": "AzureSqlSink", + "preCopyScript": { + "value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}", + "type": "Expression" + }, + "tableOption": "autoCreate", + "disableMetricsCollection": false + }, + "enableStaging": false, + "translator": { + "type": "TabularTranslator", + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + } + } + }, + "inputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.SourceConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "AnyDatabaseTable", + "type": "DatasetReference", + "parameters": { + "LinkedServiceConnectionSecret": { + "value": "@pipeline().parameters.TargetConnectionSecret", + "type": "Expression" + }, + "SchemaName": { + "value": "@item().SchemaName", + "type": "Expression" + }, + "TableName": { + "value": "@item().TableName", + "type": "Expression" + } + } + } + ] + } + ] + } + } + ], + "parameters": { + "SourceConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb01" + }, + "TargetConnectionSecret": { + "type": "string", + "defaultValue": "ConnectionString-trainingdb02" + }, + "TableLookupQuery": { + "type": "string", + "defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'" + } + }, + "folder": { + "name": "Demo Pipelines/Dynamic Linked Services" + }, + "annotations": [], + "lastPublishTime": "2020-08-27T09:43:42Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Order Summary with Mapping.json b/Code/DataFactory/pipeline/Order Summary with Mapping.json new file mode 100644 index 0000000..fda2ce8 --- /dev/null +++ b/Code/DataFactory/pipeline/Order Summary with Mapping.json @@ -0,0 +1,37 @@ +{ + "name": "Order Summary with Mapping", + "properties": { + "activities": [ + { + "name": "Mapping Order Aggregation", + "type": "ExecuteDataFlow", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataflow": { + "referenceName": "MappingOrderAggregation", + "type": "DataFlowReference" + }, + "compute": { + "coreCount": 8, + "computeType": "General" + }, + "traceLevel": "Fine" + } + } + ], + "folder": { + "name": "Demo Pipelines/Data Flows" + }, + "annotations": [], + "lastPublishTime": "2020-09-02T14:13:15Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Scale Out Level 1.json b/Code/DataFactory/pipeline/Scale Out Level 1.json new file mode 100644 index 0000000..524d9a2 --- /dev/null +++ b/Code/DataFactory/pipeline/Scale Out Level 1.json @@ -0,0 +1,179 @@ +{ + "name": "Scale Out Level 1", + "properties": { + "activities": [ + { + "name": "Get Bucket Ids", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Reset And Log Start", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Reset Bucket Process Map", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[dbo].[GetBuckets]", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference" + }, + "firstRowOnly": false + } + }, + { + "name": "Call Buckets", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Ids", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Ids').output.value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 50, + "activities": [ + { + "name": "Call Level 2", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "Scale Out Level 2", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "BucketId": { + "value": "@{item().BucketId}", + "type": "Expression" + } + } + } + } + ] + } + }, + { + "name": "Log End", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Call Buckets", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Reset And Log Start", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[dbo].[SetBucketLogEntry]", + "storedProcedureParameters": { + "LogId": { + "value": { + "value": "@activity('Reset And Log Start').output.firstRow.LogId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + }, + { + "name": "Reset And Log Start", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[dbo].[SetBucketLogEntry]", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference" + } + } + }, + { + "name": "Reset Bucket Process Map", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[dbo].[SetBucketProcesses]" + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ], + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/Scale Out Level 2.json b/Code/DataFactory/pipeline/Scale Out Level 2.json new file mode 100644 index 0000000..d4ea721 --- /dev/null +++ b/Code/DataFactory/pipeline/Scale Out Level 2.json @@ -0,0 +1,97 @@ +{ + "name": "Scale Out Level 2", + "properties": { + "activities": [ + { + "name": "Get Bucket Contents", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[dbo].[GetBucketContents]", + "storedProcedureParameters": { + "BucketId": { + "type": "Int32", + "value": { + "value": "@pipeline().parameters.BucketId", + "type": "Expression" + } + } + }, + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference" + }, + "firstRowOnly": false + } + }, + { + "name": "Execute Processes", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Bucket Contents", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Bucket Contents').output.value", + "type": "Expression" + }, + "batchCount": 50, + "activities": [ + { + "name": "Run Process", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": { + "value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "trainingdb01", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "parameters": { + "BucketId": { + "type": "int" + } + }, + "folder": { + "name": "Demo Pipelines/Scaling Out" + }, + "annotations": [], + "lastPublishTime": "2020-08-24T13:41:19Z" + }, + "type": "Microsoft.DataFactory/factories/pipelines" +} \ No newline at end of file