dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Repo reorg

Browse Source
This commit is contained in:
Paul Andrew
2022-08-16 08:42:57 +01:00
parent d969bf2580
commit dca62f67e7
34 changed files with 119 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Labs/adf/pipeline/BuildProductDimension.json
View File

@@ -1,36 +0,0 @@
{
"name": "BuildProductDimension",
"properties": {
"activities": [
{
"name": "BuildProductDimension",
"type": "ExecuteDataFlow",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataflow": {
"referenceName": "BuildProductDimension",
"type": "DataFlowReference"
},
"compute": {
"coreCount": 8,
"computeType": "General"
}
}
}
],
"folder": {
"name": "Lab4"
},
"annotations": [],
"lastPublishTime": "2020-09-26T17:05:06Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

83
Labs/adf/pipeline/CopyAdventureWorks.json
View File

@@ -1,83 +0,0 @@
{
"name": "CopyAdventureWorks",
"properties": {
"activities": [
{
"name": "Get table list",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": "SELECT TABLE_SCHEMA, TABLE_NAME \nFROM INFORMATION_SCHEMA.TABLES\nWHERE TABLE_TYPE = 'BASE TABLE'\nAND TABLE_SCHEMA = 'SalesLT'\n",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "ASQL_Product",
"type": "DatasetReference"
},
"firstRowOnly": false
}
},
{
"name": "ForEach1",
"type": "ForEach",
"dependsOn": [
{
"activity": "Get table list",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('Get table list').output.value",
"type": "Expression"
},
"activities": [
{
"name": "Execute Pipeline1",
"type": "ExecutePipeline",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "CopyAnyTable",
"type": "PipelineReference"
},
"waitOnCompletion": true,
"parameters": {
"TableSchema": {
"value": "@item().TABLE_SCHEMA",
"type": "Expression"
},
"TableName": {
"value": "@item().TABLE_NAME",
"type": "Expression"
}
}
}
}
]
}
}
],
"folder": {
"name": "Lab5"
},
"annotations": [],
"lastPublishTime": "2020-09-26T17:11:52Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

89
Labs/adf/pipeline/CopyAnyTable.json
View File

@@ -1,89 +0,0 @@
{
"name": "CopyAnyTable",
"properties": {
"activities": [
{
"name": "Copy table",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "ASQL_AdventureWorks_Generic",
"type": "DatasetReference",
"parameters": {
"TableSchema": {
"value": "@pipeline().parameters.TableSchema",
"type": "Expression"
},
"TableName": {
"value": "@pipeline().parameters.TableName",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "ADLS_Raw_Generic",
"type": "DatasetReference",
"parameters": {
"File": {
"value": "@{pipeline().parameters.TableSchema}.@{pipeline().parameters.TableName}.txt",
"type": "Expression"
}
}
}
]
}
],
"parameters": {
"TableSchema": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "Lab5"
},
"annotations": [],
"lastPublishTime": "2020-09-26T17:11:52Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

65
Labs/adf/pipeline/CopyProduct.json
View File

@@ -1,65 +0,0 @@
{
"name": "CopyProduct",
"properties": {
"activities": [
{
"name": "Copy data to lake",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "ASQL_Product",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_Raw",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab2"
},
"annotations": [],
"lastPublishTime": "2020-09-26T14:33:50Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

38
Labs/adf/pipeline/CopyProductCategory.json
View File

@@ -1,38 +0,0 @@
{
"name": "CopyProductCategory",
"properties": {
"activities": [
{
"name": "Execute CopyProductCategory",
"type": "ExecuteSSISPackage",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"packageLocation": {
"packagePath": "ADayFullOfAdf/SsisPackages/CopyProductCategory.dtsx",
"type": "SSISDB"
},
"environmentPath": null,
"connectVia": {
"referenceName": "SSISIntegrationRuntime",
"type": "IntegrationRuntimeReference"
},
"loggingLevel": "Basic"
}
}
],
"folder": {
"name": "Lab3"
},
"annotations": [],
"lastPublishTime": "2020-09-26T15:27:22Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}

65
Labs/adf/pipeline/CopyProductModel.json
View File

@@ -1,65 +0,0 @@
{
"name": "CopyProductModel",
"properties": {
"activities": [
{
"name": "Copy data1",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"sink": {
"type": "DelimitedTextSink",
"storeSettings": {
"type": "AzureBlobFSWriteSettings"
},
"formatSettings": {
"type": "DelimitedTextWriteSettings",
"quoteAllText": true,
"fileExtension": ".txt"
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"typeConversion": true,
"typeConversionSettings": {
"allowDataTruncation": true,
"treatBooleanAsNumber": false
}
}
},
"inputs": [
{
"referenceName": "ASQL_ProductModel",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "ADLS_Raw",
"type": "DatasetReference"
}
]
}
],
"folder": {
"name": "Lab4"
},
"annotations": [],
"lastPublishTime": "2020-09-26T16:27:34Z"
},
"type": "Microsoft.DataFactory/factories/pipelines"
}