dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adding credential: TrainingCredential01

Browse Source 
Adding dataflow: MappingOrderAggregation
Adding dataset: LakeFileOrderHeaderParquet
Adding dataset: LakeFileOrderDetailLinesParquet
Adding dataset: TableOrderSummary
Adding linkedService: traininglake01noneKV
Adding linkedService: trainingdb01
Adding linkedService: TrainingKeys01
Adding dataflow: WranglingOrderAggregation
Adding dataset: GetSetMetadata
Adding dataset: AnyDatabaseTable
Adding linkedService: AnyDatabaseConnection
Adding dataset: LaptopFolders
Adding linkedService: LaptopFiles
Adding integrationRuntime: PaulsLaptopIR
Adding dataset: LaptopFilePersonCSV
Adding dataset: LakeFiles
Adding linkedService: traininglake01
Adding dataset: LaptopsFiles
Adding dataset: LakeFilePersonCSV
Adding dataset: LakeFileParquet
Adding integrationRuntime: ForDataFlowDemos
Adding integrationRuntime: VNetEnabledIR
Adding managedVirtualNetwork: default
Adding linkedService: traininglak01
Adding linkedService: TrainingKeys01withUMI
Adding linkedService: TrainingStore01
Adding linkedService: BatchForTraining01
Adding pipeline: Scale Out Level 2
Adding pipeline: Scale Out Level 1
Adding pipeline: Lazy Replication
Adding pipeline: 03 - Upload - From Discovery
Adding pipeline: 01 - Upload - Simple
Adding pipeline: 04 - Upload - From Metadata
Adding pipeline: Order Summary with Mapping
Adding pipeline: 02 - Upload - Copy Params
Adding pipeline: 1950 Activities
Adding pipeline: Custom Activity
Adding managedPrivateEndpoint: AzureKeyVault1
Adding factory: TrainingFactoryDev
This commit is contained in:
Paul Andrew 2022-02-11 10:15:02 +00:00
parent 21254cf18e
commit be14d332c0
40 changed files with 2268 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Code/DataFactory/credential/TrainingCredential01.json Normal file
View File

@ -0,0 +1,9 @@
{
"name": "TrainingCredential01",
"properties": {
"type": "ManagedIdentity",
"typeProperties": {
"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
}
}
}

42
Code/DataFactory/dataflow/MappingOrderAggregation.json Normal file
View File

@ -0,0 +1,42 @@
{
"name": "MappingOrderAggregation",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "LakeFileOrderHeaderParquet",
"type": "DatasetReference"
},
"name": "OrderHeader"
},
{
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
},
"name": "OrderLineDetails"
}
],
"sinks": [
{
"dataset": {
"referenceName": "TableOrderSummary",
"type": "DatasetReference"
},
"name": "OrderSummary"
}
],
"transformations": [
{
"name": "JoinHeaderToLineDetails"
},
{
"name": "OrderLineCount"
}
],
"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
}
}
}

19
Code/DataFactory/dataflow/WranglingOrderAggregation.json Normal file
View File

@ -0,0 +1,19 @@
{
"name": "WranglingOrderAggregation",
"properties": {
"type": "WranglingDataFlow",
"typeProperties": {
"sources": [
{
"name": "LakeFileOrderDetailLinesParquet",
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
}
}
],
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
}
}
}

43
Code/DataFactory/dataset/AnyDatabaseTable.json Normal file
View File

@ -0,0 +1,43 @@
{
"name": "AnyDatabaseTable",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnection",
"type": "LinkedServiceReference",
"parameters": {
"DBConnectionSecret": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

16
Code/DataFactory/dataset/GetSetMetadata.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "GetSetMetadata",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

68
Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json Normal file
View File

@ -0,0 +1,68 @@
{
"name": "LakeFileOrderDetailLinesParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderDetailLines.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "SalesOrderDetailID",
"type": "INT32"
},
{
"name": "OrderQty",
"type": "INT32"
},
{
"name": "ProductID",
"type": "INT32"
},
{
"name": "UnitPrice",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "UnitPriceDiscount",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "LineTotal",
"type": "DECIMAL",
"precision": 38,
"scale": 6
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

122
Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json Normal file
View File

@ -0,0 +1,122 @@
{
"name": "LakeFileOrderHeaderParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderHeader.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "RevisionNumber",
"type": "INT32"
},
{
"name": "OrderDate",
"type": "INT96"
},
{
"name": "DueDate",
"type": "INT96"
},
{
"name": "ShipDate",
"type": "INT96"
},
{
"name": "Status",
"type": "INT32"
},
{
"name": "OnlineOrderFlag",
"type": "BOOLEAN"
},
{
"name": "SalesOrderNumber",
"type": "UTF8"
},
{
"name": "PurchaseOrderNumber",
"type": "UTF8"
},
{
"name": "AccountNumber",
"type": "UTF8"
},
{
"name": "CustomerID",
"type": "INT32"
},
{
"name": "ShipToAddressID",
"type": "INT32"
},
{
"name": "BillToAddressID",
"type": "INT32"
},
{
"name": "ShipMethod",
"type": "UTF8"
},
{
"name": "CreditCardApprovalCode",
"type": "UTF8"
},
{
"name": "SubTotal",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TaxAmt",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Freight",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TotalDue",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Comment",
"type": "UTF8"
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

39
Code/DataFactory/dataset/LakeFileParquet.json Normal file
View File

@ -0,0 +1,39 @@
{
"name": "LakeFileParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

37
Code/DataFactory/dataset/LakeFilePersonCSV.json Normal file
View File

@ -0,0 +1,37 @@
{
"name": "LakeFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Person.csv",
"folderPath": "Landing",
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

41
Code/DataFactory/dataset/LakeFiles.json Normal file
View File

@ -0,0 +1,41 @@
{
"name": "LakeFiles",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

36
Code/DataFactory/dataset/LaptopFilePersonCSV.json Normal file
View File

@ -0,0 +1,36 @@
{
"name": "LaptopFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "Person.csv",
"folderPath": "ForUpload/People"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

33
Code/DataFactory/dataset/LaptopFolders.json Normal file
View File

@ -0,0 +1,33 @@
{
"name": "LaptopFolders",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

40
Code/DataFactory/dataset/LaptopsFiles.json Normal file
View File

@ -0,0 +1,40 @@
{
"name": "LaptopsFiles",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

30
Code/DataFactory/dataset/TableOrderSummary.json Normal file
View File

@ -0,0 +1,30 @@
{
"name": "TableOrderSummary",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "SalesOrderNumber",
"type": "varchar"
},
{
"name": "RecordCount",
"type": "int",
"precision": 10
}
],
"typeProperties": {
"schema": "dbo",
"table": "OrderSummary"
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

15
Code/DataFactory/factory/TrainingFactoryDev.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "TrainingFactoryDev",
"location": "uksouth",
"identity": {
"type": "SystemAssigned,UserAssigned",
"principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f",
"tenantId": "929039b9-c19d-425e-993b-1e3fda155876",
"userAssignedIdentities": {
"/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": {
"clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85",
"principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91"
}
}
}
}

16
Code/DataFactory/integrationRuntime/ForDataFlowDemos.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "ForDataFlowDemos",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 240
}
}
}
}
}

12
Code/DataFactory/integrationRuntime/PaulsLaptopIR.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "PaulsLaptopIR",
"properties": {
"type": "SelfHosted",
"typeProperties": {
"linkedInfo": {
"resourceId": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WorkLaptop",
"authorizationType": "Rbac"
}
}
}
}

21
Code/DataFactory/integrationRuntime/VNetEnabledIR.json Normal file
View File

@ -0,0 +1,21 @@
{
"name": "VNetEnabledIR",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 0,
"cleanup": true
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
}
}

26
Code/DataFactory/linkedService/AnyDatabaseConnection.json Normal file
View File

@ -0,0 +1,26 @@
{
"name": "AnyDatabaseConnection",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"parameters": {
"DBConnectionSecret": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": {
"value": "@linkedService().DBConnectionSecret",
"type": "Expression"
}
}
}
}
}

17
Code/DataFactory/linkedService/BatchForTraining01.json Normal file
View File

@ -0,0 +1,17 @@
{
"name": "BatchForTraining01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBatch",
"typeProperties": {
"batchUri": "https://batchfortraining01.uksouth.batch.azure.com",
"poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654",
"accountName": "batchfortraining01",
"linkedServiceName": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
}
}
}
}

24
Code/DataFactory/linkedService/LaptopFiles.json Normal file
View File

@ -0,0 +1,24 @@
{
"name": "LaptopFiles",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "FileServer",
"typeProperties": {
"host": "C:\\ADFRoot\\",
"userId": "ADFIRUser",
"password": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ADFIRUser"
}
},
"connectVia": {
"referenceName": "PaulsLaptopIR",
"type": "IntegrationRuntimeReference"
}
}
}

11
Code/DataFactory/linkedService/TrainingKeys01.json Normal file
View File

@ -0,0 +1,11 @@
{
"name": "TrainingKeys01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "https://TrainingKeys01.vault.azure.net/"
}
}
}

15
Code/DataFactory/linkedService/TrainingKeys01withUMI.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "TrainingKeys01withUMI",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "https://TrainingKeys01.vault.azure.net/",
"credential": {
"referenceName": "TrainingCredential01",
"type": "CredentialReference"
}
}
}
}

12
Code/DataFactory/linkedService/TrainingStore01.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "TrainingStore01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobStorage",
"typeProperties": {
"connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9"
}
}
}

18
Code/DataFactory/linkedService/trainingdb01.json Normal file
View File

@ -0,0 +1,18 @@
{
"name": "trainingdb01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ConnectionString-trainingdb01"
}
}
}
}

12
Code/DataFactory/linkedService/traininglak01.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "traininglak01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9"
}
}
}

19
Code/DataFactory/linkedService/traininglake01.json Normal file
View File

@ -0,0 +1,19 @@
{
"name": "traininglake01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
}
}

12
Code/DataFactory/linkedService/traininglake01noneKV.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "traininglake01noneKV",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9"
}
}
}

4
Code/DataFactory/managedVirtualNetwork/default.json Normal file
View File

@ -0,0 +1,4 @@
{
"name": "default",
"type": "Microsoft.DataFactory/factories/managedvirtualnetworks"
}

10
Code/DataFactory/managedVirtualNetwork/default/managedPrivateEndpoint/AzureKeyVault1.json Normal file
View File

@ -0,0 +1,10 @@
{
"name": "AzureKeyVault1",
"properties": {
"privateLinkResourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01",
"groupId": "vault",
"fqdns": [
"trainingkeys01.vaultcore.azure.net"
]
}
}

71
Code/DataFactory/pipeline/01 - Upload - Simple.json Normal file
View File

@ -0,0 +1,71 @@
{
"name": "01 - Upload - Simple",
"properties": {
"activities": [
{
"name": "Upload Person",
"description": "hfdhgfdhfd",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopFilePersonCSV",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "LakeFilePersonCSV",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-09-29T13:50:28Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

78
Code/DataFactory/pipeline/02 - Upload - Copy Params.json Normal file
View File

@ -0,0 +1,78 @@
{
"name": "02 - Upload - Copy Params",
"properties": {
"activities": [
{
"name": "Upload Any File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": "Person.csv"
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": "Person.csv"
}
}
]
}
],
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:46Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

137
Code/DataFactory/pipeline/03 - Upload - From Discovery.json Normal file
View File

@ -0,0 +1,137 @@
{
"name": "03 - Upload - From Discovery",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "GetMetadata",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "LaptopFolders",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People"
}
},
"fieldList": [
"childItems"
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
}
},
{
"name": "Upload Files",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.childItems",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "ForUpload\\People",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": "Landing",
"File": {
"value": "@item().name",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:06:45Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

266
Code/DataFactory/pipeline/04 - Upload - From Metadata.json Normal file
View File

@ -0,0 +1,266 @@
{
"name": "04 - Upload - From Metadata",
"properties": {
"activities": [
{
"name": "Get File List",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "UploadFiles",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Log Upload Start",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get File List').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Upload File",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "DelimitedTextSource",
"additionalColumns": [
{
"name": "FilePath",
"value": "$$FILEPATH"
},
{
"name": "DataFactoryName",
"value": {
"value": "@pipeline().DataFactory",
"type": "Expression"
}
},
{
"name": "PipelineName",
"value": {
"value": "@pipeline().Pipeline",
"type": "Expression"
}
},
{
"name": "UploadDateTime",
"value": {
"value": "@utcnow()",
"type": "Expression"
}
},
{
"name": "RunId",
"value": {
"value": "@pipeline().RunId",
"type": "Expression"
}
}
],
"storeSettings": {
"type": "FileServerReadSettings",
"recursive": true
},
"formatSettings": {
"type": "DelimitedTextReadSettings"
}
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "LaptopsFiles",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().SourceDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "LakeFiles",
"type": "DatasetReference",
"parameters": {
"Directory": {
"value": "@item().TargetDirectory",
"type": "Expression"
},
"File": {
"value": "@item().FileName",
"type": "Expression"
}
}
}
]
}
]
}
},
{
"name": "Log Upload Start",
"type": "Lookup",
"dependsOn": [
{
"activity": "Get File List",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[dbo].[LogUploadStart]",
"storedProcedureParameters": {
"FileCount": {
"type": "Int32",
"value": {
"value": "@activity('Get File List').output.count",
"type": "Expression"
}
},
"TriggerId": {
"type": "String",
"value": {
"value": "@pipeline().TriggerId",
"type": "Expression"
}
}
},
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference"
}
}
},
{
"name": "Log Upload Finish",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "UploadFiles",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[dbo].[LogUploadFinish]",
"storedProcedureParameters": {
"LogId": {
"value": {
"value": "@activity('Log Upload Start').output.firstRow.LogId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
}
],
"variables": {
"DemoVariable": {
"type": "String"
}
},
"folder": {
"name": "Demo Pipelines/Data Uploads"
},
"annotations": [],
"lastPublishTime": "2020-09-02T14:13:14Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

379
Code/DataFactory/pipeline/1950 Activities.json Normal file
View File

@ -0,0 +1,379 @@
{
"name": "1950 Activities",
"properties": {
"activities": [
{
"name": "50",
"type": "ForEach",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')",
"type": "Expression"
},
"batchCount": 50,
"activities": [
{
"name": "Wait 1 Minute 1",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 2",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 3",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 4",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 5",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 6",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 7",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 8",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 9",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 10",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 11",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 12",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 13",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 14",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 15",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 16",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 17",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 18",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 19",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 20",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 21",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 22",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 23",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 24",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 25",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 26",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 27",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 28",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 29",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 30",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 31",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 32",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 33",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 34",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 35",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 36",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 37",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 38",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
},
{
"name": "Wait 1 Minute 39",
"type": "Wait",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"waitTimeInSeconds": 600
}
}
]
}
}
],
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": [],
"lastPublishTime": "2022-02-03T14:39:03Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

42
Code/DataFactory/pipeline/Custom Activity.json Normal file
View File

@ -0,0 +1,42 @@
{
"name": "Custom Activity",
"properties": {
"activities": [
{
"name": "HelloWorld",
"type": "Custom",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"command": "CustomActivity.exe",
"resourceLinkedService": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
},
"folderPath": "customactivities",
"referenceObjects": {
"linkedServices": [],
"datasets": []
}
},
"linkedServiceName": {
"referenceName": "BatchForTraining01",
"type": "LinkedServiceReference"
}
}
],
"folder": {
"name": "Demo Pipelines/Misc"
},
"annotations": [],
"lastPublishTime": "2022-02-03T14:39:04Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

163
Code/DataFactory/pipeline/Lazy Replication.json Normal file
View File

@ -0,0 +1,163 @@
{
"name": "Lazy Replication",
"properties": {
"activities": [
{
"name": "Get Table List",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": {
"value": "@pipeline().parameters.TableLookupQuery",
"type": "Expression"
},
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "AnyDatabaseTable",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": {
"value": "@pipeline().parameters.SourceConnectionSecret",
"type": "Expression"
},
"SchemaName": "sys",
"TableName": "objects"
}
},
"firstRowOnly": false
}
},
{
"name": "Copy Tables",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Table List",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Table List').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Copy Table",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "AzureSqlSink",
"preCopyScript": {
"value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}",
"type": "Expression"
},
"tableOption": "autoCreate",
"disableMetricsCollection": false
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "AnyDatabaseTable",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": {
"value": "@pipeline().parameters.SourceConnectionSecret",
"type": "Expression"
},
"SchemaName": {
"value": "@item().SchemaName",
"type": "Expression"
},
"TableName": {
"value": "@item().TableName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "AnyDatabaseTable",
"type": "DatasetReference",
"parameters": {
"LinkedServiceConnectionSecret": {
"value": "@pipeline().parameters.TargetConnectionSecret",
"type": "Expression"
},
"SchemaName": {
"value": "@item().SchemaName",
"type": "Expression"
},
"TableName": {
"value": "@item().TableName",
"type": "Expression"
}
}
}
]
}
]
}
}
],
"parameters": {
"SourceConnectionSecret": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb01"
},
"TargetConnectionSecret": {
"type": "string",
"defaultValue": "ConnectionString-trainingdb02"
},
"TableLookupQuery": {
"type": "string",
"defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'"
}
},
"folder": {
"name": "Demo Pipelines/Dynamic Linked Services"
},
"annotations": [],
"lastPublishTime": "2020-08-27T09:43:42Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

37
Code/DataFactory/pipeline/Order Summary with Mapping.json Normal file
View File

@ -0,0 +1,37 @@
{
"name": "Order Summary with Mapping",
"properties": {
"activities": [
{
"name": "Mapping Order Aggregation",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "MappingOrderAggregation",
"type": "DataFlowReference"
},
"compute": {
"coreCount": 8,
"computeType": "General"
},
"traceLevel": "Fine"
}
}
],
"folder": {
"name": "Demo Pipelines/Data Flows"
},
"annotations": [],
"lastPublishTime": "2020-09-02T14:13:15Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

179
Code/DataFactory/pipeline/Scale Out Level 1.json Normal file
View File

@ -0,0 +1,179 @@
{
"name": "Scale Out Level 1",
"properties": {
"activities": [
{
"name": "Get Bucket Ids",
"type": "Lookup",
"dependsOn": [
{
"activity": "Reset And Log Start",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Reset Bucket Process Map",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[dbo].[GetBuckets]",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "Call Buckets",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Bucket Ids",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Bucket Ids').output.value",
"type": "Expression"
},
"isSequential": false,
"batchCount": 50,
"activities": [
{
"name": "Call Level 2",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "Scale Out Level 2",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"BucketId": {
"value": "@{item().BucketId}",
"type": "Expression"
}
}
}
}
]
}
},
{
"name": "Log End",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "Call Buckets",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "Reset And Log Start",
"dependencyConditions": [
"Completed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[dbo].[SetBucketLogEntry]",
"storedProcedureParameters": {
"LogId": {
"value": {
"value": "@activity('Reset And Log Start').output.firstRow.LogId",
"type": "Expression"
},
"type": "Int32"
}
}
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
},
{
"name": "Reset And Log Start",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[dbo].[SetBucketLogEntry]",
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference"
}
}
},
{
"name": "Reset Bucket Process Map",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[dbo].[SetBucketProcesses]"
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
}
],
"folder": {
"name": "Demo Pipelines/Scaling Out"
},
"annotations": [],
"lastPublishTime": "2020-08-24T13:41:19Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

97
Code/DataFactory/pipeline/Scale Out Level 2.json Normal file
View File

@ -0,0 +1,97 @@
{
"name": "Scale Out Level 2",
"properties": {
"activities": [
{
"name": "Get Bucket Contents",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderStoredProcedureName": "[dbo].[GetBucketContents]",
"storedProcedureParameters": {
"BucketId": {
"type": "Int32",
"value": {
"value": "@pipeline().parameters.BucketId",
"type": "Expression"
}
}
},
"partitionOption": "None"
},
"dataset": {
"referenceName": "GetSetMetadata",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "Execute Processes",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get Bucket Contents",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get Bucket Contents').output.value",
"type": "Expression"
},
"batchCount": 50,
"activities": [
{
"name": "Run Process",
"type": "SqlServerStoredProcedure",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": {
"value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')",
"type": "Expression"
}
},
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
}
}
]
}
}
],
"parameters": {
"BucketId": {
"type": "int"
}
},
"folder": {
"name": "Demo Pipelines/Scaling Out"
},
"annotations": [],
"lastPublishTime": "2020-08-24T13:41:19Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}