Adding credential: TrainingCredential01
Adding dataflow: MappingOrderAggregation Adding dataset: LakeFileOrderHeaderParquet Adding dataset: LakeFileOrderDetailLinesParquet Adding dataset: TableOrderSummary Adding linkedService: traininglake01noneKV Adding linkedService: trainingdb01 Adding linkedService: TrainingKeys01 Adding dataflow: WranglingOrderAggregation Adding dataset: GetSetMetadata Adding dataset: AnyDatabaseTable Adding linkedService: AnyDatabaseConnection Adding dataset: LaptopFolders Adding linkedService: LaptopFiles Adding integrationRuntime: PaulsLaptopIR Adding dataset: LaptopFilePersonCSV Adding dataset: LakeFiles Adding linkedService: traininglake01 Adding dataset: LaptopsFiles Adding dataset: LakeFilePersonCSV Adding dataset: LakeFileParquet Adding integrationRuntime: ForDataFlowDemos Adding integrationRuntime: VNetEnabledIR Adding managedVirtualNetwork: default Adding linkedService: traininglak01 Adding linkedService: TrainingKeys01withUMI Adding linkedService: TrainingStore01 Adding linkedService: BatchForTraining01 Adding pipeline: Scale Out Level 2 Adding pipeline: Scale Out Level 1 Adding pipeline: Lazy Replication Adding pipeline: 03 - Upload - From Discovery Adding pipeline: 01 - Upload - Simple Adding pipeline: 04 - Upload - From Metadata Adding pipeline: Order Summary with Mapping Adding pipeline: 02 - Upload - Copy Params Adding pipeline: 1950 Activities Adding pipeline: Custom Activity Adding managedPrivateEndpoint: AzureKeyVault1 Adding factory: TrainingFactoryDev
This commit is contained in:
Paul Andrew
parent
21254cf18e
commit
be14d332c0
9
Code/DataFactory/credential/TrainingCredential01.json
Normal file
9
Code/DataFactory/credential/TrainingCredential01.json
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"name": "TrainingCredential01",
|
||||||
|
"properties": {
|
||||||
|
"type": "ManagedIdentity",
|
||||||
|
"typeProperties": {
|
||||||
|
"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
42
Code/DataFactory/dataflow/MappingOrderAggregation.json
Normal file
42
Code/DataFactory/dataflow/MappingOrderAggregation.json
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
"name": "MappingOrderAggregation",
|
||||||
|
"properties": {
|
||||||
|
"type": "MappingDataFlow",
|
||||||
|
"typeProperties": {
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "LakeFileOrderHeaderParquet",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
},
|
||||||
|
"name": "OrderHeader"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "LakeFileOrderDetailLinesParquet",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
},
|
||||||
|
"name": "OrderLineDetails"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"sinks": [
|
||||||
|
{
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "TableOrderSummary",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
},
|
||||||
|
"name": "OrderSummary"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"transformations": [
|
||||||
|
{
|
||||||
|
"name": "JoinHeaderToLineDetails"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "OrderLineCount"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
Code/DataFactory/dataflow/WranglingOrderAggregation.json
Normal file
19
Code/DataFactory/dataflow/WranglingOrderAggregation.json
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"name": "WranglingOrderAggregation",
|
||||||
|
"properties": {
|
||||||
|
"type": "WranglingDataFlow",
|
||||||
|
"typeProperties": {
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"name": "LakeFileOrderDetailLinesParquet",
|
||||||
|
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "LakeFileOrderDetailLinesParquet",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
43
Code/DataFactory/dataset/AnyDatabaseTable.json
Normal file
43
Code/DataFactory/dataset/AnyDatabaseTable.json
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
{
|
||||||
|
"name": "AnyDatabaseTable",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "AnyDatabaseConnection",
|
||||||
|
"type": "LinkedServiceReference",
|
||||||
|
"parameters": {
|
||||||
|
"DBConnectionSecret": {
|
||||||
|
"value": "@dataset().LinkedServiceConnectionSecret",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parameters": {
|
||||||
|
"LinkedServiceConnectionSecret": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"SchemaName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"TableName": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "SQLDB"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureSqlTable",
|
||||||
|
"schema": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"schema": {
|
||||||
|
"value": "@dataset().SchemaName",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"table": {
|
||||||
|
"value": "@dataset().TableName",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
16
Code/DataFactory/dataset/GetSetMetadata.json
Normal file
16
Code/DataFactory/dataset/GetSetMetadata.json
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"name": "GetSetMetadata",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "trainingdb01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "SQLDB"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureSqlTable",
|
||||||
|
"schema": []
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
@ -0,0 +1,68 @@
|
|||||||
|
{
|
||||||
|
"name": "LakeFileOrderDetailLinesParquet",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "traininglake01noneKV",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Lake"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "Parquet",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "AzureBlobFSLocation",
|
||||||
|
"fileName": "OrderDetailLines.parquet",
|
||||||
|
"folderPath": "Raw",
|
||||||
|
"fileSystem": "datawarehouse"
|
||||||
|
},
|
||||||
|
"compressionCodec": "snappy"
|
||||||
|
},
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"name": "SalesOrderID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "SalesOrderDetailID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "OrderQty",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ProductID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "UnitPrice",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 19,
|
||||||
|
"scale": 4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "UnitPriceDiscount",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 19,
|
||||||
|
"scale": 4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "LineTotal",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 38,
|
||||||
|
"scale": 6
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "rowguid",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ModifiedDate",
|
||||||
|
"type": "INT96"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
122
Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json
Normal file
122
Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
{
|
||||||
|
"name": "LakeFileOrderHeaderParquet",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "traininglake01noneKV",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Lake"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "Parquet",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "AzureBlobFSLocation",
|
||||||
|
"fileName": "OrderHeader.parquet",
|
||||||
|
"folderPath": "Raw",
|
||||||
|
"fileSystem": "datawarehouse"
|
||||||
|
},
|
||||||
|
"compressionCodec": "snappy"
|
||||||
|
},
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"name": "SalesOrderID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "RevisionNumber",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "OrderDate",
|
||||||
|
"type": "INT96"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DueDate",
|
||||||
|
"type": "INT96"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ShipDate",
|
||||||
|
"type": "INT96"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Status",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "OnlineOrderFlag",
|
||||||
|
"type": "BOOLEAN"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "SalesOrderNumber",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "PurchaseOrderNumber",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "AccountNumber",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "CustomerID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ShipToAddressID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "BillToAddressID",
|
||||||
|
"type": "INT32"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ShipMethod",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "CreditCardApprovalCode",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "SubTotal",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 19,
|
||||||
|
"scale": 4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "TaxAmt",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 19,
|
||||||
|
"scale": 4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Freight",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 19,
|
||||||
|
"scale": 4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "TotalDue",
|
||||||
|
"type": "DECIMAL",
|
||||||
|
"precision": 19,
|
||||||
|
"scale": 4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Comment",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "rowguid",
|
||||||
|
"type": "UTF8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ModifiedDate",
|
||||||
|
"type": "INT96"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
39
Code/DataFactory/dataset/LakeFileParquet.json
Normal file
39
Code/DataFactory/dataset/LakeFileParquet.json
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"name": "LakeFileParquet",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "traininglake01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"parameters": {
|
||||||
|
"Directory": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"File": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Lake"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "Parquet",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "AzureBlobFSLocation",
|
||||||
|
"fileName": {
|
||||||
|
"value": "@dataset().File",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"folderPath": {
|
||||||
|
"value": "@dataset().Directory",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"fileSystem": "datawarehouse"
|
||||||
|
},
|
||||||
|
"compressionCodec": "snappy"
|
||||||
|
},
|
||||||
|
"schema": []
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
37
Code/DataFactory/dataset/LakeFilePersonCSV.json
Normal file
37
Code/DataFactory/dataset/LakeFilePersonCSV.json
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"name": "LakeFilePersonCSV",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "traininglake01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Lake"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "DelimitedText",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "AzureBlobFSLocation",
|
||||||
|
"fileName": "Person.csv",
|
||||||
|
"folderPath": "Landing",
|
||||||
|
"fileSystem": "datawarehouse"
|
||||||
|
},
|
||||||
|
"columnDelimiter": ",",
|
||||||
|
"escapeChar": "\\",
|
||||||
|
"quoteChar": "\""
|
||||||
|
},
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"type": "String"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "String"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "String"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
41
Code/DataFactory/dataset/LakeFiles.json
Normal file
41
Code/DataFactory/dataset/LakeFiles.json
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"name": "LakeFiles",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "traininglake01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"parameters": {
|
||||||
|
"Directory": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"File": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Lake"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "DelimitedText",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "AzureBlobFSLocation",
|
||||||
|
"fileName": {
|
||||||
|
"value": "@dataset().File",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"folderPath": {
|
||||||
|
"value": "@dataset().Directory",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"fileSystem": "datawarehouse"
|
||||||
|
},
|
||||||
|
"columnDelimiter": ",",
|
||||||
|
"escapeChar": "\\",
|
||||||
|
"quoteChar": "\""
|
||||||
|
},
|
||||||
|
"schema": []
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
36
Code/DataFactory/dataset/LaptopFilePersonCSV.json
Normal file
36
Code/DataFactory/dataset/LaptopFilePersonCSV.json
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"name": "LaptopFilePersonCSV",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "LaptopFiles",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Laptop"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "DelimitedText",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "FileServerLocation",
|
||||||
|
"fileName": "Person.csv",
|
||||||
|
"folderPath": "ForUpload/People"
|
||||||
|
},
|
||||||
|
"columnDelimiter": ",",
|
||||||
|
"escapeChar": "\\",
|
||||||
|
"quoteChar": "\""
|
||||||
|
},
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"type": "String"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "String"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "String"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
33
Code/DataFactory/dataset/LaptopFolders.json
Normal file
33
Code/DataFactory/dataset/LaptopFolders.json
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"name": "LaptopFolders",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "LaptopFiles",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"parameters": {
|
||||||
|
"Directory": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Laptop"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "DelimitedText",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "FileServerLocation",
|
||||||
|
"folderPath": {
|
||||||
|
"value": "@dataset().Directory",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"columnDelimiter": ",",
|
||||||
|
"escapeChar": "\\",
|
||||||
|
"quoteChar": "\""
|
||||||
|
},
|
||||||
|
"schema": []
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
40
Code/DataFactory/dataset/LaptopsFiles.json
Normal file
40
Code/DataFactory/dataset/LaptopsFiles.json
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
{
|
||||||
|
"name": "LaptopsFiles",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "LaptopFiles",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"parameters": {
|
||||||
|
"Directory": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"File": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Laptop"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "DelimitedText",
|
||||||
|
"typeProperties": {
|
||||||
|
"location": {
|
||||||
|
"type": "FileServerLocation",
|
||||||
|
"fileName": {
|
||||||
|
"value": "@dataset().File",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"folderPath": {
|
||||||
|
"value": "@dataset().Directory",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"columnDelimiter": ",",
|
||||||
|
"escapeChar": "\\",
|
||||||
|
"quoteChar": "\""
|
||||||
|
},
|
||||||
|
"schema": []
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
30
Code/DataFactory/dataset/TableOrderSummary.json
Normal file
30
Code/DataFactory/dataset/TableOrderSummary.json
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"name": "TableOrderSummary",
|
||||||
|
"properties": {
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "trainingdb01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "SQLDB"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureSqlTable",
|
||||||
|
"schema": [
|
||||||
|
{
|
||||||
|
"name": "SalesOrderNumber",
|
||||||
|
"type": "varchar"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "RecordCount",
|
||||||
|
"type": "int",
|
||||||
|
"precision": 10
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"typeProperties": {
|
||||||
|
"schema": "dbo",
|
||||||
|
"table": "OrderSummary"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/datasets"
|
||||||
|
}
|
||||||
15
Code/DataFactory/factory/TrainingFactoryDev.json
Normal file
15
Code/DataFactory/factory/TrainingFactoryDev.json
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"name": "TrainingFactoryDev",
|
||||||
|
"location": "uksouth",
|
||||||
|
"identity": {
|
||||||
|
"type": "SystemAssigned,UserAssigned",
|
||||||
|
"principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f",
|
||||||
|
"tenantId": "929039b9-c19d-425e-993b-1e3fda155876",
|
||||||
|
"userAssignedIdentities": {
|
||||||
|
"/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": {
|
||||||
|
"clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85",
|
||||||
|
"principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
16
Code/DataFactory/integrationRuntime/ForDataFlowDemos.json
Normal file
16
Code/DataFactory/integrationRuntime/ForDataFlowDemos.json
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"name": "ForDataFlowDemos",
|
||||||
|
"properties": {
|
||||||
|
"type": "Managed",
|
||||||
|
"typeProperties": {
|
||||||
|
"computeProperties": {
|
||||||
|
"location": "UK South",
|
||||||
|
"dataFlowProperties": {
|
||||||
|
"computeType": "General",
|
||||||
|
"coreCount": 8,
|
||||||
|
"timeToLive": 240
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
12
Code/DataFactory/integrationRuntime/PaulsLaptopIR.json
Normal file
12
Code/DataFactory/integrationRuntime/PaulsLaptopIR.json
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"name": "PaulsLaptopIR",
|
||||||
|
"properties": {
|
||||||
|
"type": "SelfHosted",
|
||||||
|
"typeProperties": {
|
||||||
|
"linkedInfo": {
|
||||||
|
"resourceId": "/subscriptions/77a3e40b-dd36-433c-90ca-a09f39724af3/resourcegroups/CommunityDemos/providers/Microsoft.DataFactory/factories/PaulsFunFactoryV2/integrationruntimes/WorkLaptop",
|
||||||
|
"authorizationType": "Rbac"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
21
Code/DataFactory/integrationRuntime/VNetEnabledIR.json
Normal file
21
Code/DataFactory/integrationRuntime/VNetEnabledIR.json
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "VNetEnabledIR",
|
||||||
|
"properties": {
|
||||||
|
"type": "Managed",
|
||||||
|
"typeProperties": {
|
||||||
|
"computeProperties": {
|
||||||
|
"location": "UK South",
|
||||||
|
"dataFlowProperties": {
|
||||||
|
"computeType": "General",
|
||||||
|
"coreCount": 8,
|
||||||
|
"timeToLive": 0,
|
||||||
|
"cleanup": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"managedVirtualNetwork": {
|
||||||
|
"type": "ManagedVirtualNetworkReference",
|
||||||
|
"referenceName": "default"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
26
Code/DataFactory/linkedService/AnyDatabaseConnection.json
Normal file
26
Code/DataFactory/linkedService/AnyDatabaseConnection.json
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"name": "AnyDatabaseConnection",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"parameters": {
|
||||||
|
"DBConnectionSecret": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureSqlDatabase",
|
||||||
|
"typeProperties": {
|
||||||
|
"connectionString": {
|
||||||
|
"type": "AzureKeyVaultSecret",
|
||||||
|
"store": {
|
||||||
|
"referenceName": "TrainingKeys01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"secretName": {
|
||||||
|
"value": "@linkedService().DBConnectionSecret",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
17
Code/DataFactory/linkedService/BatchForTraining01.json
Normal file
17
Code/DataFactory/linkedService/BatchForTraining01.json
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"name": "BatchForTraining01",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureBatch",
|
||||||
|
"typeProperties": {
|
||||||
|
"batchUri": "https://batchfortraining01.uksouth.batch.azure.com",
|
||||||
|
"poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654",
|
||||||
|
"accountName": "batchfortraining01",
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "TrainingStore01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
24
Code/DataFactory/linkedService/LaptopFiles.json
Normal file
24
Code/DataFactory/linkedService/LaptopFiles.json
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"name": "LaptopFiles",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "FileServer",
|
||||||
|
"typeProperties": {
|
||||||
|
"host": "C:\\ADFRoot\\",
|
||||||
|
"userId": "ADFIRUser",
|
||||||
|
"password": {
|
||||||
|
"type": "AzureKeyVaultSecret",
|
||||||
|
"store": {
|
||||||
|
"referenceName": "TrainingKeys01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"secretName": "ADFIRUser"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"connectVia": {
|
||||||
|
"referenceName": "PaulsLaptopIR",
|
||||||
|
"type": "IntegrationRuntimeReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
11
Code/DataFactory/linkedService/TrainingKeys01.json
Normal file
11
Code/DataFactory/linkedService/TrainingKeys01.json
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"name": "TrainingKeys01",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureKeyVault",
|
||||||
|
"typeProperties": {
|
||||||
|
"baseUrl": "https://TrainingKeys01.vault.azure.net/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
15
Code/DataFactory/linkedService/TrainingKeys01withUMI.json
Normal file
15
Code/DataFactory/linkedService/TrainingKeys01withUMI.json
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"name": "TrainingKeys01withUMI",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureKeyVault",
|
||||||
|
"typeProperties": {
|
||||||
|
"baseUrl": "https://TrainingKeys01.vault.azure.net/",
|
||||||
|
"credential": {
|
||||||
|
"referenceName": "TrainingCredential01",
|
||||||
|
"type": "CredentialReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
12
Code/DataFactory/linkedService/TrainingStore01.json
Normal file
12
Code/DataFactory/linkedService/TrainingStore01.json
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"name": "TrainingStore01",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureBlobStorage",
|
||||||
|
"typeProperties": {
|
||||||
|
"connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;",
|
||||||
|
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
18
Code/DataFactory/linkedService/trainingdb01.json
Normal file
18
Code/DataFactory/linkedService/trainingdb01.json
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"name": "trainingdb01",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureSqlDatabase",
|
||||||
|
"typeProperties": {
|
||||||
|
"connectionString": {
|
||||||
|
"type": "AzureKeyVaultSecret",
|
||||||
|
"store": {
|
||||||
|
"referenceName": "TrainingKeys01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"secretName": "ConnectionString-trainingdb01"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
12
Code/DataFactory/linkedService/traininglak01.json
Normal file
12
Code/DataFactory/linkedService/traininglak01.json
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"name": "traininglak01",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureBlobFS",
|
||||||
|
"typeProperties": {
|
||||||
|
"url": "https://traininglake01.dfs.core.windows.net",
|
||||||
|
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
Code/DataFactory/linkedService/traininglake01.json
Normal file
19
Code/DataFactory/linkedService/traininglake01.json
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"name": "traininglake01",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureBlobFS",
|
||||||
|
"typeProperties": {
|
||||||
|
"url": "https://traininglake01.dfs.core.windows.net",
|
||||||
|
"accountKey": {
|
||||||
|
"type": "AzureKeyVaultSecret",
|
||||||
|
"store": {
|
||||||
|
"referenceName": "TrainingKeys01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"secretName": "traininglake01"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
12
Code/DataFactory/linkedService/traininglake01noneKV.json
Normal file
12
Code/DataFactory/linkedService/traininglake01noneKV.json
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"name": "traininglake01noneKV",
|
||||||
|
"type": "Microsoft.DataFactory/factories/linkedservices",
|
||||||
|
"properties": {
|
||||||
|
"annotations": [],
|
||||||
|
"type": "AzureBlobFS",
|
||||||
|
"typeProperties": {
|
||||||
|
"url": "https://traininglake01.dfs.core.windows.net",
|
||||||
|
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
4
Code/DataFactory/managedVirtualNetwork/default.json
Normal file
4
Code/DataFactory/managedVirtualNetwork/default.json
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"name": "default",
|
||||||
|
"type": "Microsoft.DataFactory/factories/managedvirtualnetworks"
|
||||||
|
}
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"name": "AzureKeyVault1",
|
||||||
|
"properties": {
|
||||||
|
"privateLinkResourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01",
|
||||||
|
"groupId": "vault",
|
||||||
|
"fqdns": [
|
||||||
|
"trainingkeys01.vaultcore.azure.net"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
71
Code/DataFactory/pipeline/01 - Upload - Simple.json
Normal file
71
Code/DataFactory/pipeline/01 - Upload - Simple.json
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
{
|
||||||
|
"name": "01 - Upload - Simple",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Upload Person",
|
||||||
|
"description": "hfdhgfdhfd",
|
||||||
|
"type": "Copy",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "DelimitedTextSource",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "FileServerReadSettings",
|
||||||
|
"recursive": true
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextReadSettings"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sink": {
|
||||||
|
"type": "DelimitedTextSink",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "AzureBlobFSWriteSettings"
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextWriteSettings",
|
||||||
|
"quoteAllText": true,
|
||||||
|
"fileExtension": ".txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"enableStaging": false,
|
||||||
|
"translator": {
|
||||||
|
"type": "TabularTranslator",
|
||||||
|
"typeConversion": true,
|
||||||
|
"typeConversionSettings": {
|
||||||
|
"allowDataTruncation": true,
|
||||||
|
"treatBooleanAsNumber": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LaptopFilePersonCSV",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LakeFilePersonCSV",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Data Uploads"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-09-29T13:50:28Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
78
Code/DataFactory/pipeline/02 - Upload - Copy Params.json
Normal file
78
Code/DataFactory/pipeline/02 - Upload - Copy Params.json
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
{
|
||||||
|
"name": "02 - Upload - Copy Params",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Upload Any File",
|
||||||
|
"type": "Copy",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "DelimitedTextSource",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "FileServerReadSettings",
|
||||||
|
"recursive": true
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextReadSettings"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sink": {
|
||||||
|
"type": "DelimitedTextSink",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "AzureBlobFSWriteSettings"
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextWriteSettings",
|
||||||
|
"quoteAllText": true,
|
||||||
|
"fileExtension": ".txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"enableStaging": false,
|
||||||
|
"translator": {
|
||||||
|
"type": "TabularTranslator",
|
||||||
|
"typeConversion": true,
|
||||||
|
"typeConversionSettings": {
|
||||||
|
"allowDataTruncation": true,
|
||||||
|
"treatBooleanAsNumber": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LaptopsFiles",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": "ForUpload\\People",
|
||||||
|
"File": "Person.csv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LakeFiles",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": "Landing",
|
||||||
|
"File": "Person.csv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Data Uploads"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-08-27T09:06:46Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
137
Code/DataFactory/pipeline/03 - Upload - From Discovery.json
Normal file
137
Code/DataFactory/pipeline/03 - Upload - From Discovery.json
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
{
|
||||||
|
"name": "03 - Upload - From Discovery",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Get File List",
|
||||||
|
"type": "GetMetadata",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "LaptopFolders",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": "ForUpload\\People"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"fieldList": [
|
||||||
|
"childItems"
|
||||||
|
],
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "FileServerReadSettings",
|
||||||
|
"recursive": true
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextReadSettings"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Upload Files",
|
||||||
|
"type": "ForEach",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Get File List",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"items": {
|
||||||
|
"value": "@activity('Get File List').output.childItems",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Upload File",
|
||||||
|
"type": "Copy",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "DelimitedTextSource",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "FileServerReadSettings",
|
||||||
|
"recursive": true
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextReadSettings"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sink": {
|
||||||
|
"type": "DelimitedTextSink",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "AzureBlobFSWriteSettings"
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextWriteSettings",
|
||||||
|
"quoteAllText": true,
|
||||||
|
"fileExtension": ".txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"enableStaging": false,
|
||||||
|
"translator": {
|
||||||
|
"type": "TabularTranslator",
|
||||||
|
"typeConversion": true,
|
||||||
|
"typeConversionSettings": {
|
||||||
|
"allowDataTruncation": true,
|
||||||
|
"treatBooleanAsNumber": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LaptopsFiles",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": "ForUpload\\People",
|
||||||
|
"File": {
|
||||||
|
"value": "@item().name",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LakeFiles",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": "Landing",
|
||||||
|
"File": {
|
||||||
|
"value": "@item().name",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Data Uploads"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-08-27T09:06:45Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
266
Code/DataFactory/pipeline/04 - Upload - From Metadata.json
Normal file
266
Code/DataFactory/pipeline/04 - Upload - From Metadata.json
Normal file
@ -0,0 +1,266 @@
|
|||||||
|
{
|
||||||
|
"name": "04 - Upload - From Metadata",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Get File List",
|
||||||
|
"type": "Lookup",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;",
|
||||||
|
"queryTimeout": "02:00:00",
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "GetSetMetadata",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
},
|
||||||
|
"firstRowOnly": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "UploadFiles",
|
||||||
|
"type": "ForEach",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Get File List",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"activity": "Log Upload Start",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"items": {
|
||||||
|
"value": "@activity('Get File List').output.value",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Upload File",
|
||||||
|
"type": "Copy",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "DelimitedTextSource",
|
||||||
|
"additionalColumns": [
|
||||||
|
{
|
||||||
|
"name": "FilePath",
|
||||||
|
"value": "$$FILEPATH"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DataFactoryName",
|
||||||
|
"value": {
|
||||||
|
"value": "@pipeline().DataFactory",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "PipelineName",
|
||||||
|
"value": {
|
||||||
|
"value": "@pipeline().Pipeline",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "UploadDateTime",
|
||||||
|
"value": {
|
||||||
|
"value": "@utcnow()",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "RunId",
|
||||||
|
"value": {
|
||||||
|
"value": "@pipeline().RunId",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "FileServerReadSettings",
|
||||||
|
"recursive": true
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextReadSettings"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sink": {
|
||||||
|
"type": "DelimitedTextSink",
|
||||||
|
"storeSettings": {
|
||||||
|
"type": "AzureBlobFSWriteSettings"
|
||||||
|
},
|
||||||
|
"formatSettings": {
|
||||||
|
"type": "DelimitedTextWriteSettings",
|
||||||
|
"quoteAllText": true,
|
||||||
|
"fileExtension": ".txt"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"enableStaging": false,
|
||||||
|
"translator": {
|
||||||
|
"type": "TabularTranslator",
|
||||||
|
"typeConversion": true,
|
||||||
|
"typeConversionSettings": {
|
||||||
|
"allowDataTruncation": true,
|
||||||
|
"treatBooleanAsNumber": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LaptopsFiles",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": {
|
||||||
|
"value": "@item().SourceDirectory",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"File": {
|
||||||
|
"value": "@item().FileName",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "LakeFiles",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"Directory": {
|
||||||
|
"value": "@item().TargetDirectory",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"File": {
|
||||||
|
"value": "@item().FileName",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Log Upload Start",
|
||||||
|
"type": "Lookup",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Get File List",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"sqlReaderStoredProcedureName": "[dbo].[LogUploadStart]",
|
||||||
|
"storedProcedureParameters": {
|
||||||
|
"FileCount": {
|
||||||
|
"type": "Int32",
|
||||||
|
"value": {
|
||||||
|
"value": "@activity('Get File List').output.count",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"TriggerId": {
|
||||||
|
"type": "String",
|
||||||
|
"value": {
|
||||||
|
"value": "@pipeline().TriggerId",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"queryTimeout": "02:00:00",
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "GetSetMetadata",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Log Upload Finish",
|
||||||
|
"type": "SqlServerStoredProcedure",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "UploadFiles",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"storedProcedureName": "[dbo].[LogUploadFinish]",
|
||||||
|
"storedProcedureParameters": {
|
||||||
|
"LogId": {
|
||||||
|
"value": {
|
||||||
|
"value": "@activity('Log Upload Start').output.firstRow.LogId",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"type": "Int32"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "trainingdb01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"variables": {
|
||||||
|
"DemoVariable": {
|
||||||
|
"type": "String"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Data Uploads"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-09-02T14:13:14Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
379
Code/DataFactory/pipeline/1950 Activities.json
Normal file
379
Code/DataFactory/pipeline/1950 Activities.json
Normal file
@ -0,0 +1,379 @@
|
|||||||
|
{
|
||||||
|
"name": "1950 Activities",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "50",
|
||||||
|
"type": "ForEach",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"items": {
|
||||||
|
"value": "@json('[\"1\",\n\"2\",\n\"3\",\n\"4\",\n\"5\",\n\"6\",\n\"7\",\n\"8\",\n\"9\",\n\"10\",\n\"11\",\n\"12\",\n\"13\",\n\"14\",\n\"15\",\n\"16\",\n\"17\",\n\"18\",\n\"19\",\n\"20\",\n\"21\",\n\"22\",\n\"23\",\n\"24\",\n\"25\",\n\"26\",\n\"27\",\n\"28\",\n\"29\",\n\"30\",\n\"31\",\n\"32\",\n\"33\",\n\"34\",\n\"35\",\n\"36\",\n\"37\",\n\"38\",\n\"39\",\n\"40\",\n\"41\",\n\"42\",\n\"43\",\n\"44\",\n\"45\",\n\"46\",\n\"47\",\n\"48\",\n\"49\",\n\"50\"]\n')",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"batchCount": 50,
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 1",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 2",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 3",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 4",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 5",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 6",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 7",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 8",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 9",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 10",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 11",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 12",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 13",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 14",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 15",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 16",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 17",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 18",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 19",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 20",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 21",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 22",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 23",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 24",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 25",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 26",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 27",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 28",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 29",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 30",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 31",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 32",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 33",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 34",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 35",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 36",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 37",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 38",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Wait 1 Minute 39",
|
||||||
|
"type": "Wait",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"waitTimeInSeconds": 600
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Misc"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2022-02-03T14:39:03Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
42
Code/DataFactory/pipeline/Custom Activity.json
Normal file
42
Code/DataFactory/pipeline/Custom Activity.json
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
"name": "Custom Activity",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "HelloWorld",
|
||||||
|
"type": "Custom",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"command": "CustomActivity.exe",
|
||||||
|
"resourceLinkedService": {
|
||||||
|
"referenceName": "TrainingStore01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
},
|
||||||
|
"folderPath": "customactivities",
|
||||||
|
"referenceObjects": {
|
||||||
|
"linkedServices": [],
|
||||||
|
"datasets": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "BatchForTraining01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Misc"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2022-02-03T14:39:04Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
163
Code/DataFactory/pipeline/Lazy Replication.json
Normal file
163
Code/DataFactory/pipeline/Lazy Replication.json
Normal file
@ -0,0 +1,163 @@
|
|||||||
|
{
|
||||||
|
"name": "Lazy Replication",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Get Table List",
|
||||||
|
"type": "Lookup",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"sqlReaderQuery": {
|
||||||
|
"value": "@pipeline().parameters.TableLookupQuery",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"queryTimeout": "02:00:00",
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "AnyDatabaseTable",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"LinkedServiceConnectionSecret": {
|
||||||
|
"value": "@pipeline().parameters.SourceConnectionSecret",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"SchemaName": "sys",
|
||||||
|
"TableName": "objects"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"firstRowOnly": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Copy Tables",
|
||||||
|
"type": "ForEach",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Get Table List",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"items": {
|
||||||
|
"value": "@activity('Get Table List').output.value",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Copy Table",
|
||||||
|
"type": "Copy",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"queryTimeout": "02:00:00",
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"sink": {
|
||||||
|
"type": "AzureSqlSink",
|
||||||
|
"preCopyScript": {
|
||||||
|
"value": "IF OBJECT_ID('@{item().SchemaName}.@{item().TableName}') IS NOT NULL TRUNCATE TABLE @{item().SchemaName}.@{item().TableName}",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"tableOption": "autoCreate",
|
||||||
|
"disableMetricsCollection": false
|
||||||
|
},
|
||||||
|
"enableStaging": false,
|
||||||
|
"translator": {
|
||||||
|
"type": "TabularTranslator",
|
||||||
|
"typeConversion": true,
|
||||||
|
"typeConversionSettings": {
|
||||||
|
"allowDataTruncation": true,
|
||||||
|
"treatBooleanAsNumber": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "AnyDatabaseTable",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"LinkedServiceConnectionSecret": {
|
||||||
|
"value": "@pipeline().parameters.SourceConnectionSecret",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"SchemaName": {
|
||||||
|
"value": "@item().SchemaName",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"TableName": {
|
||||||
|
"value": "@item().TableName",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"referenceName": "AnyDatabaseTable",
|
||||||
|
"type": "DatasetReference",
|
||||||
|
"parameters": {
|
||||||
|
"LinkedServiceConnectionSecret": {
|
||||||
|
"value": "@pipeline().parameters.TargetConnectionSecret",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"SchemaName": {
|
||||||
|
"value": "@item().SchemaName",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"TableName": {
|
||||||
|
"value": "@item().TableName",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"SourceConnectionSecret": {
|
||||||
|
"type": "string",
|
||||||
|
"defaultValue": "ConnectionString-trainingdb01"
|
||||||
|
},
|
||||||
|
"TargetConnectionSecret": {
|
||||||
|
"type": "string",
|
||||||
|
"defaultValue": "ConnectionString-trainingdb02"
|
||||||
|
},
|
||||||
|
"TableLookupQuery": {
|
||||||
|
"type": "string",
|
||||||
|
"defaultValue": "SELECT \ts.name AS SchemaName, \to.name AS TableName FROM \tsys.objects o \tINNER JOIN sys.schemas s \t\tON o.schema_id = s.schema_id WHERE \to.[type] = 'U'"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Dynamic Linked Services"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-08-27T09:43:42Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
37
Code/DataFactory/pipeline/Order Summary with Mapping.json
Normal file
37
Code/DataFactory/pipeline/Order Summary with Mapping.json
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"name": "Order Summary with Mapping",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Mapping Order Aggregation",
|
||||||
|
"type": "ExecuteDataFlow",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"dataflow": {
|
||||||
|
"referenceName": "MappingOrderAggregation",
|
||||||
|
"type": "DataFlowReference"
|
||||||
|
},
|
||||||
|
"compute": {
|
||||||
|
"coreCount": 8,
|
||||||
|
"computeType": "General"
|
||||||
|
},
|
||||||
|
"traceLevel": "Fine"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Data Flows"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-09-02T14:13:15Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
179
Code/DataFactory/pipeline/Scale Out Level 1.json
Normal file
179
Code/DataFactory/pipeline/Scale Out Level 1.json
Normal file
@ -0,0 +1,179 @@
|
|||||||
|
{
|
||||||
|
"name": "Scale Out Level 1",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Get Bucket Ids",
|
||||||
|
"type": "Lookup",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Reset And Log Start",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"activity": "Reset Bucket Process Map",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"sqlReaderStoredProcedureName": "[dbo].[GetBuckets]",
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "GetSetMetadata",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
},
|
||||||
|
"firstRowOnly": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Call Buckets",
|
||||||
|
"type": "ForEach",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Get Bucket Ids",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"items": {
|
||||||
|
"value": "@activity('Get Bucket Ids').output.value",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"isSequential": false,
|
||||||
|
"batchCount": 50,
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Call Level 2",
|
||||||
|
"type": "ExecutePipeline",
|
||||||
|
"dependsOn": [],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"pipeline": {
|
||||||
|
"referenceName": "Scale Out Level 2",
|
||||||
|
"type": "PipelineReference"
|
||||||
|
},
|
||||||
|
"waitOnCompletion": true,
|
||||||
|
"parameters": {
|
||||||
|
"BucketId": {
|
||||||
|
"value": "@{item().BucketId}",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Log End",
|
||||||
|
"type": "SqlServerStoredProcedure",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Call Buckets",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"activity": "Reset And Log Start",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Completed"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"storedProcedureName": "[dbo].[SetBucketLogEntry]",
|
||||||
|
"storedProcedureParameters": {
|
||||||
|
"LogId": {
|
||||||
|
"value": {
|
||||||
|
"value": "@activity('Reset And Log Start').output.firstRow.LogId",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"type": "Int32"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "trainingdb01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Reset And Log Start",
|
||||||
|
"type": "Lookup",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"sqlReaderStoredProcedureName": "[dbo].[SetBucketLogEntry]",
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "GetSetMetadata",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Reset Bucket Process Map",
|
||||||
|
"type": "SqlServerStoredProcedure",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"storedProcedureName": "[dbo].[SetBucketProcesses]"
|
||||||
|
},
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "trainingdb01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Scaling Out"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-08-24T13:41:19Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
97
Code/DataFactory/pipeline/Scale Out Level 2.json
Normal file
97
Code/DataFactory/pipeline/Scale Out Level 2.json
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
{
|
||||||
|
"name": "Scale Out Level 2",
|
||||||
|
"properties": {
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Get Bucket Contents",
|
||||||
|
"type": "Lookup",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"source": {
|
||||||
|
"type": "AzureSqlSource",
|
||||||
|
"sqlReaderStoredProcedureName": "[dbo].[GetBucketContents]",
|
||||||
|
"storedProcedureParameters": {
|
||||||
|
"BucketId": {
|
||||||
|
"type": "Int32",
|
||||||
|
"value": {
|
||||||
|
"value": "@pipeline().parameters.BucketId",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"partitionOption": "None"
|
||||||
|
},
|
||||||
|
"dataset": {
|
||||||
|
"referenceName": "GetSetMetadata",
|
||||||
|
"type": "DatasetReference"
|
||||||
|
},
|
||||||
|
"firstRowOnly": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Execute Processes",
|
||||||
|
"type": "ForEach",
|
||||||
|
"dependsOn": [
|
||||||
|
{
|
||||||
|
"activity": "Get Bucket Contents",
|
||||||
|
"dependencyConditions": [
|
||||||
|
"Succeeded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"items": {
|
||||||
|
"value": "@activity('Get Bucket Contents').output.value",
|
||||||
|
"type": "Expression"
|
||||||
|
},
|
||||||
|
"batchCount": 50,
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"name": "Run Process",
|
||||||
|
"type": "SqlServerStoredProcedure",
|
||||||
|
"dependsOn": [],
|
||||||
|
"policy": {
|
||||||
|
"timeout": "7.00:00:00",
|
||||||
|
"retry": 0,
|
||||||
|
"retryIntervalInSeconds": 30,
|
||||||
|
"secureOutput": false,
|
||||||
|
"secureInput": false
|
||||||
|
},
|
||||||
|
"userProperties": [],
|
||||||
|
"typeProperties": {
|
||||||
|
"storedProcedureName": {
|
||||||
|
"value": "@concat('[workers].[DumpDataAndWait',item().ProcessId,']')",
|
||||||
|
"type": "Expression"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"linkedServiceName": {
|
||||||
|
"referenceName": "trainingdb01",
|
||||||
|
"type": "LinkedServiceReference"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"BucketId": {
|
||||||
|
"type": "int"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"folder": {
|
||||||
|
"name": "Demo Pipelines/Scaling Out"
|
||||||
|
},
|
||||||
|
"annotations": [],
|
||||||
|
"lastPublishTime": "2020-08-24T13:41:19Z"
|
||||||
|
},
|
||||||
|
"type": "Microsoft.DataFactory/factories/pipelines"
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user