dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adding credential: TrainingCredential01

Browse Source 
Adding dataflow: MappingOrderAggregation
Adding dataset: LakeFileOrderHeaderParquet
Adding dataset: LakeFileOrderDetailLinesParquet
Adding dataset: TableOrderSummary
Adding linkedService: traininglake01noneKV
Adding linkedService: trainingdb01
Adding linkedService: TrainingKeys01
Adding dataflow: WranglingOrderAggregation
Adding dataset: GetSetMetadata
Adding dataset: AnyDatabaseTable
Adding linkedService: AnyDatabaseConnection
Adding dataset: LaptopFolders
Adding linkedService: LaptopFiles
Adding integrationRuntime: PaulsLaptopIR
Adding dataset: LaptopFilePersonCSV
Adding dataset: LakeFiles
Adding linkedService: traininglake01
Adding dataset: LaptopsFiles
Adding dataset: LakeFilePersonCSV
Adding dataset: LakeFileParquet
Adding integrationRuntime: ForDataFlowDemos
Adding integrationRuntime: VNetEnabledIR
Adding managedVirtualNetwork: default
Adding linkedService: traininglak01
Adding linkedService: TrainingKeys01withUMI
Adding linkedService: TrainingStore01
Adding linkedService: BatchForTraining01
Adding pipeline: Scale Out Level 2
Adding pipeline: Scale Out Level 1
Adding pipeline: Lazy Replication
Adding pipeline: 03 - Upload - From Discovery
Adding pipeline: 01 - Upload - Simple
Adding pipeline: 04 - Upload - From Metadata
Adding pipeline: Order Summary with Mapping
Adding pipeline: 02 - Upload - Copy Params
Adding pipeline: 1950 Activities
Adding pipeline: Custom Activity
Adding managedPrivateEndpoint: AzureKeyVault1
Adding factory: TrainingFactoryDev
This commit is contained in:
Paul Andrew
2022-02-11 10:15:02 +00:00
parent 21254cf18e
commit be14d332c0
40 changed files with 2268 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
Code/DataFactory/dataset/AnyDatabaseTable.json Normal file
View File

@@ -0,0 +1,43 @@
{
"name": "AnyDatabaseTable",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnection",
"type": "LinkedServiceReference",
"parameters": {
"DBConnectionSecret": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

16
Code/DataFactory/dataset/GetSetMetadata.json Normal file
View File

@@ -0,0 +1,16 @@
{
"name": "GetSetMetadata",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

68
Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json Normal file
View File

@@ -0,0 +1,68 @@
{
"name": "LakeFileOrderDetailLinesParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderDetailLines.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "SalesOrderDetailID",
"type": "INT32"
},
{
"name": "OrderQty",
"type": "INT32"
},
{
"name": "ProductID",
"type": "INT32"
},
{
"name": "UnitPrice",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "UnitPriceDiscount",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "LineTotal",
"type": "DECIMAL",
"precision": 38,
"scale": 6
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

122
Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json Normal file
View File

@@ -0,0 +1,122 @@
{
"name": "LakeFileOrderHeaderParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderHeader.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "RevisionNumber",
"type": "INT32"
},
{
"name": "OrderDate",
"type": "INT96"
},
{
"name": "DueDate",
"type": "INT96"
},
{
"name": "ShipDate",
"type": "INT96"
},
{
"name": "Status",
"type": "INT32"
},
{
"name": "OnlineOrderFlag",
"type": "BOOLEAN"
},
{
"name": "SalesOrderNumber",
"type": "UTF8"
},
{
"name": "PurchaseOrderNumber",
"type": "UTF8"
},
{
"name": "AccountNumber",
"type": "UTF8"
},
{
"name": "CustomerID",
"type": "INT32"
},
{
"name": "ShipToAddressID",
"type": "INT32"
},
{
"name": "BillToAddressID",
"type": "INT32"
},
{
"name": "ShipMethod",
"type": "UTF8"
},
{
"name": "CreditCardApprovalCode",
"type": "UTF8"
},
{
"name": "SubTotal",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TaxAmt",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Freight",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TotalDue",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Comment",
"type": "UTF8"
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

39
Code/DataFactory/dataset/LakeFileParquet.json Normal file
View File

@@ -0,0 +1,39 @@
{
"name": "LakeFileParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

37
Code/DataFactory/dataset/LakeFilePersonCSV.json Normal file
View File

@@ -0,0 +1,37 @@
{
"name": "LakeFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Person.csv",
"folderPath": "Landing",
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

41
Code/DataFactory/dataset/LakeFiles.json Normal file
View File

@@ -0,0 +1,41 @@
{
"name": "LakeFiles",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

36
Code/DataFactory/dataset/LaptopFilePersonCSV.json Normal file
View File

@@ -0,0 +1,36 @@
{
"name": "LaptopFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "Person.csv",
"folderPath": "ForUpload/People"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

33
Code/DataFactory/dataset/LaptopFolders.json Normal file
View File

@@ -0,0 +1,33 @@
{
"name": "LaptopFolders",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

40
Code/DataFactory/dataset/LaptopsFiles.json Normal file
View File

@@ -0,0 +1,40 @@
{
"name": "LaptopsFiles",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

30
Code/DataFactory/dataset/TableOrderSummary.json Normal file
View File

@@ -0,0 +1,30 @@
{
"name": "TableOrderSummary",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "SalesOrderNumber",
"type": "varchar"
},
{
"name": "RecordCount",
"type": "int",
"precision": 10
}
],
"typeProperties": {
"schema": "dbo",
"table": "OrderSummary"
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}