From 213168013bc605cb93aa2a71ffe0ade54bb71fb2 Mon Sep 17 00:00:00 2001 From: Richard Swinbank Date: Mon, 29 Aug 2022 21:32:02 +0100 Subject: [PATCH] Removed factory artifacts --- .../dataflow/BuildProductDimension.json | 64 ----------- Code/DataFactory/dataset/ADLS_DimProduct.json | 24 ---- Code/DataFactory/dataset/ADLS_Product.json | 28 ----- .../dataset/ADLS_ProductCategory.json | 28 ----- .../dataset/ADLS_ProductModel.json | 28 ----- Code/DataFactory/dataset/ADLS_Raw.json | 27 ----- .../DataFactory/dataset/ADLS_Raw_Generic.json | 36 ------ .../dataset/ASQL_AdventureWorks_Generic.json | 34 ------ Code/DataFactory/dataset/ASQL_Product.json | 106 ------------------ .../dataset/ASQL_ProductModel.json | 106 ------------------ .../pipeline/BuildProductDimension.json | 36 ------ .../pipeline/CopyAdventureWorks.json | 83 -------------- Code/DataFactory/pipeline/CopyAnyTable.json | 89 --------------- Code/DataFactory/pipeline/CopyProduct.json | 65 ----------- .../pipeline/CopyProductCategory.json | 38 ------- .../pipeline/CopyProductModel.json | 65 ----------- 16 files changed, 857 deletions(-) delete mode 100644 Code/DataFactory/dataflow/BuildProductDimension.json delete mode 100644 Code/DataFactory/dataset/ADLS_DimProduct.json delete mode 100644 Code/DataFactory/dataset/ADLS_Product.json delete mode 100644 Code/DataFactory/dataset/ADLS_ProductCategory.json delete mode 100644 Code/DataFactory/dataset/ADLS_ProductModel.json delete mode 100644 Code/DataFactory/dataset/ADLS_Raw.json delete mode 100644 Code/DataFactory/dataset/ADLS_Raw_Generic.json delete mode 100644 Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json delete mode 100644 Code/DataFactory/dataset/ASQL_Product.json delete mode 100644 Code/DataFactory/dataset/ASQL_ProductModel.json delete mode 100644 Code/DataFactory/pipeline/BuildProductDimension.json delete mode 100644 Code/DataFactory/pipeline/CopyAdventureWorks.json delete mode 100644 Code/DataFactory/pipeline/CopyAnyTable.json delete mode 100644 Code/DataFactory/pipeline/CopyProduct.json delete mode 100644 Code/DataFactory/pipeline/CopyProductCategory.json delete mode 100644 Code/DataFactory/pipeline/CopyProductModel.json diff --git a/Code/DataFactory/dataflow/BuildProductDimension.json b/Code/DataFactory/dataflow/BuildProductDimension.json deleted file mode 100644 index 503aa5f..0000000 --- a/Code/DataFactory/dataflow/BuildProductDimension.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "BuildProductDimension", - "properties": { - "folder": { - "name": "Lab4" - }, - "type": "MappingDataFlow", - "typeProperties": { - "sources": [ - { - "dataset": { - "referenceName": "ADLS_Product", - "type": "DatasetReference" - }, - "name": "Product" - }, - { - "dataset": { - "referenceName": "ADLS_ProductCategory", - "type": "DatasetReference" - }, - "name": "ProductCategory" - }, - { - "dataset": { - "referenceName": "ADLS_ProductModel", - "type": "DatasetReference" - }, - "name": "ProductModel" - } - ], - "sinks": [ - { - "dataset": { - "referenceName": "ADLS_DimProduct", - "type": "DatasetReference" - }, - "name": "DimProduct" - } - ], - "transformations": [ - { - "name": "SelectProductColumns" - }, - { - "name": "SelectProductCategoryColumns" - }, - { - "name": "SelectProductModelColumns" - }, - { - "name": "LookupProductCategory" - }, - { - "name": "LookupProductModelID" - }, - { - "name": "RemoveDuplicateColumns" - } - ], - "script": "source(output(\n\t\tProductID as short,\n\t\tName as string,\n\t\tProductNumber as string,\n\t\tColor as string,\n\t\tStandardCost as double,\n\t\tListPrice as double,\n\t\tSize as string,\n\t\tWeight as double,\n\t\tProductCategoryID as short,\n\t\tProductModelID as short,\n\t\tSellStartDate as string,\n\t\tSellEndDate as string,\n\t\tDiscontinuedDate as string,\n\t\tThumbNailPhoto as string,\n\t\tThumbnailPhotoFileName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> Product\nsource(output(\n\t\tProductCategoryID as short,\n\t\tParentProductCategoryID as string,\n\t\tName as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductCategory\nsource(output(\n\t\tProductModelID as short,\n\t\tName as string,\n\t\tCatalogDescription as string,\n\t\trowguid as string,\n\t\tModifiedDate as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> ProductModel\nProduct select(mapColumn(\n\t\tProductID,\n\t\tProduct = Name,\n\t\tProductCategoryID,\n\t\tProductModelID\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductColumns\nProductCategory select(mapColumn(\n\t\tProductCategoryID,\n\t\tProductCategory = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductCategoryColumns\nProductModel select(mapColumn(\n\t\tProductModelID,\n\t\tProductModel = Name\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectProductModelColumns\nSelectProductColumns, SelectProductCategoryColumns lookup(SelectProductColumns@ProductCategoryID == SelectProductCategoryColumns@ProductCategoryID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductCategory\nLookupProductCategory, SelectProductModelColumns lookup(SelectProductColumns@ProductModelID == SelectProductModelColumns@ProductModelID,\n\tmultiple: false,\n\tpickup: 'any',\n\tbroadcast: 'auto')~> LookupProductModelID\nLookupProductModelID select(mapColumn(\n\t\tProductID,\n\t\tProduct,\n\t\tProductCategoryID = {SelectProductColumns@ProductCategoryID},\n\t\tProductModelID = {SelectProductColumns@ProductModelID},\n\t\tProductCategory,\n\t\tProductModel\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns\nRemoveDuplicateColumns sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'parquet',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> DimProduct" - } - } -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_DimProduct.json b/Code/DataFactory/dataset/ADLS_DimProduct.json deleted file mode 100644 index dcbdb00..0000000 --- a/Code/DataFactory/dataset/ADLS_DimProduct.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "ADLS_DimProduct", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "Parquet", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "folderPath": "Cleaned/DimProduct", - "fileSystem": "lakeroot" - }, - "compressionCodec": "snappy" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_Product.json b/Code/DataFactory/dataset/ADLS_Product.json deleted file mode 100644 index ab9a7b3..0000000 --- a/Code/DataFactory/dataset/ADLS_Product.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ADLS_Product", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": "SalesLT.Product.txt", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_ProductCategory.json b/Code/DataFactory/dataset/ADLS_ProductCategory.json deleted file mode 100644 index e357684..0000000 --- a/Code/DataFactory/dataset/ADLS_ProductCategory.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ADLS_ProductCategory", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": "SalesLT.ProductCategory.txt", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_ProductModel.json b/Code/DataFactory/dataset/ADLS_ProductModel.json deleted file mode 100644 index 013aea5..0000000 --- a/Code/DataFactory/dataset/ADLS_ProductModel.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ADLS_ProductModel", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": "SalesLT.ProductModel.txt", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_Raw.json b/Code/DataFactory/dataset/ADLS_Raw.json deleted file mode 100644 index 251c768..0000000 --- a/Code/DataFactory/dataset/ADLS_Raw.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "ADLS_Raw", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab2" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ADLS_Raw_Generic.json b/Code/DataFactory/dataset/ADLS_Raw_Generic.json deleted file mode 100644 index e9d2448..0000000 --- a/Code/DataFactory/dataset/ADLS_Raw_Generic.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "ADLS_Raw_Generic", - "properties": { - "linkedServiceName": { - "referenceName": "AzureDataLakeStorage1", - "type": "LinkedServiceReference" - }, - "parameters": { - "File": { - "type": "string" - } - }, - "folder": { - "name": "Lab5" - }, - "annotations": [], - "type": "DelimitedText", - "typeProperties": { - "location": { - "type": "AzureBlobFSLocation", - "fileName": { - "value": "@dataset().File", - "type": "Expression" - }, - "folderPath": "Raw", - "fileSystem": "lakeroot" - }, - "columnDelimiter": ",", - "escapeChar": "\\", - "firstRowAsHeader": true, - "quoteChar": "\"" - }, - "schema": [] - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json b/Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json deleted file mode 100644 index 8339d81..0000000 --- a/Code/DataFactory/dataset/ASQL_AdventureWorks_Generic.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "ASQL_AdventureWorks_Generic", - "properties": { - "linkedServiceName": { - "referenceName": "AzureSqlDatabase1", - "type": "LinkedServiceReference" - }, - "parameters": { - "TableSchema": { - "type": "string" - }, - "TableName": { - "type": "string" - } - }, - "folder": { - "name": "Lab5" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [], - "typeProperties": { - "schema": { - "value": "@dataset().TableSchema", - "type": "Expression" - }, - "table": { - "value": "@dataset().TableName", - "type": "Expression" - } - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ASQL_Product.json b/Code/DataFactory/dataset/ASQL_Product.json deleted file mode 100644 index ebe1cc1..0000000 --- a/Code/DataFactory/dataset/ASQL_Product.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "name": "ASQL_Product", - "properties": { - "linkedServiceName": { - "referenceName": "AzureSqlDatabase1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab2" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [ - { - "name": "ProductID", - "type": "int", - "precision": 10 - }, - { - "name": "Name", - "type": "nvarchar" - }, - { - "name": "ProductNumber", - "type": "nvarchar" - }, - { - "name": "Color", - "type": "nvarchar" - }, - { - "name": "StandardCost", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "ListPrice", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "Size", - "type": "nvarchar" - }, - { - "name": "Weight", - "type": "decimal", - "precision": 8, - "scale": 2 - }, - { - "name": "ProductCategoryID", - "type": "int", - "precision": 10 - }, - { - "name": "ProductModelID", - "type": "int", - "precision": 10 - }, - { - "name": "SellStartDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "SellEndDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "DiscontinuedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "ThumbNailPhoto", - "type": "varbinary" - }, - { - "name": "ThumbnailPhotoFileName", - "type": "nvarchar" - }, - { - "name": "rowguid", - "type": "uniqueidentifier" - }, - { - "name": "ModifiedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - } - ], - "typeProperties": { - "schema": "SalesLT", - "table": "Product" - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/dataset/ASQL_ProductModel.json b/Code/DataFactory/dataset/ASQL_ProductModel.json deleted file mode 100644 index be402d9..0000000 --- a/Code/DataFactory/dataset/ASQL_ProductModel.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "name": "ASQL_ProductModel", - "properties": { - "linkedServiceName": { - "referenceName": "AzureSqlDatabase1", - "type": "LinkedServiceReference" - }, - "folder": { - "name": "Lab4" - }, - "annotations": [], - "type": "AzureSqlTable", - "schema": [ - { - "name": "ProductID", - "type": "int", - "precision": 10 - }, - { - "name": "Name", - "type": "nvarchar" - }, - { - "name": "ProductNumber", - "type": "nvarchar" - }, - { - "name": "Color", - "type": "nvarchar" - }, - { - "name": "StandardCost", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "ListPrice", - "type": "money", - "precision": 19, - "scale": 4 - }, - { - "name": "Size", - "type": "nvarchar" - }, - { - "name": "Weight", - "type": "decimal", - "precision": 8, - "scale": 2 - }, - { - "name": "ProductCategoryID", - "type": "int", - "precision": 10 - }, - { - "name": "ProductModelID", - "type": "int", - "precision": 10 - }, - { - "name": "SellStartDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "SellEndDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "DiscontinuedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - }, - { - "name": "ThumbNailPhoto", - "type": "varbinary" - }, - { - "name": "ThumbnailPhotoFileName", - "type": "nvarchar" - }, - { - "name": "rowguid", - "type": "uniqueidentifier" - }, - { - "name": "ModifiedDate", - "type": "datetime", - "precision": 23, - "scale": 3 - } - ], - "typeProperties": { - "schema": "SalesLT", - "table": "ProductModel" - } - }, - "type": "Microsoft.DataFactory/factories/datasets" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/BuildProductDimension.json b/Code/DataFactory/pipeline/BuildProductDimension.json deleted file mode 100644 index ff36156..0000000 --- a/Code/DataFactory/pipeline/BuildProductDimension.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "BuildProductDimension", - "properties": { - "activities": [ - { - "name": "BuildProductDimension", - "type": "ExecuteDataFlow", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "dataflow": { - "referenceName": "BuildProductDimension", - "type": "DataFlowReference" - }, - "compute": { - "coreCount": 8, - "computeType": "General" - } - } - } - ], - "folder": { - "name": "Lab4" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T17:05:06Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAdventureWorks.json b/Code/DataFactory/pipeline/CopyAdventureWorks.json deleted file mode 100644 index cf3a0b6..0000000 --- a/Code/DataFactory/pipeline/CopyAdventureWorks.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "name": "CopyAdventureWorks", - "properties": { - "activities": [ - { - "name": "Get table list", - "type": "Lookup", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "sqlReaderQuery": "SELECT TABLE_SCHEMA, TABLE_NAME \nFROM INFORMATION_SCHEMA.TABLES\nWHERE TABLE_TYPE = 'BASE TABLE'\nAND TABLE_SCHEMA = 'SalesLT'\n", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "dataset": { - "referenceName": "ASQL_Product", - "type": "DatasetReference" - }, - "firstRowOnly": false - } - }, - { - "name": "ForEach1", - "type": "ForEach", - "dependsOn": [ - { - "activity": "Get table list", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "userProperties": [], - "typeProperties": { - "items": { - "value": "@activity('Get table list').output.value", - "type": "Expression" - }, - "activities": [ - { - "name": "Execute Pipeline1", - "type": "ExecutePipeline", - "dependsOn": [], - "userProperties": [], - "typeProperties": { - "pipeline": { - "referenceName": "CopyAnyTable", - "type": "PipelineReference" - }, - "waitOnCompletion": true, - "parameters": { - "TableSchema": { - "value": "@item().TABLE_SCHEMA", - "type": "Expression" - }, - "TableName": { - "value": "@item().TABLE_NAME", - "type": "Expression" - } - } - } - } - ] - } - } - ], - "folder": { - "name": "Lab5" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T17:11:52Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyAnyTable.json b/Code/DataFactory/pipeline/CopyAnyTable.json deleted file mode 100644 index 2992669..0000000 --- a/Code/DataFactory/pipeline/CopyAnyTable.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "name": "CopyAnyTable", - "properties": { - "activities": [ - { - "name": "Copy table", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "ASQL_AdventureWorks_Generic", - "type": "DatasetReference", - "parameters": { - "TableSchema": { - "value": "@pipeline().parameters.TableSchema", - "type": "Expression" - }, - "TableName": { - "value": "@pipeline().parameters.TableName", - "type": "Expression" - } - } - } - ], - "outputs": [ - { - "referenceName": "ADLS_Raw_Generic", - "type": "DatasetReference", - "parameters": { - "File": { - "value": "@{pipeline().parameters.TableSchema}.@{pipeline().parameters.TableName}.txt", - "type": "Expression" - } - } - } - ] - } - ], - "parameters": { - "TableSchema": { - "type": "string" - }, - "TableName": { - "type": "string" - } - }, - "folder": { - "name": "Lab5" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T17:11:52Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyProduct.json b/Code/DataFactory/pipeline/CopyProduct.json deleted file mode 100644 index 70cae29..0000000 --- a/Code/DataFactory/pipeline/CopyProduct.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "CopyProduct", - "properties": { - "activities": [ - { - "name": "Copy data to lake", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "ASQL_Product", - "type": "DatasetReference" - } - ], - "outputs": [ - { - "referenceName": "ADLS_Raw", - "type": "DatasetReference" - } - ] - } - ], - "folder": { - "name": "Lab2" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T14:33:50Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyProductCategory.json b/Code/DataFactory/pipeline/CopyProductCategory.json deleted file mode 100644 index 08290d9..0000000 --- a/Code/DataFactory/pipeline/CopyProductCategory.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "CopyProductCategory", - "properties": { - "activities": [ - { - "name": "Execute CopyProductCategory", - "type": "ExecuteSSISPackage", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "packageLocation": { - "packagePath": "ADayFullOfAdf/SsisPackages/CopyProductCategory.dtsx", - "type": "SSISDB" - }, - "environmentPath": null, - "connectVia": { - "referenceName": "SSISIntegrationRuntime", - "type": "IntegrationRuntimeReference" - }, - "loggingLevel": "Basic" - } - } - ], - "folder": { - "name": "Lab3" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T15:27:22Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file diff --git a/Code/DataFactory/pipeline/CopyProductModel.json b/Code/DataFactory/pipeline/CopyProductModel.json deleted file mode 100644 index d19be52..0000000 --- a/Code/DataFactory/pipeline/CopyProductModel.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "CopyProductModel", - "properties": { - "activities": [ - { - "name": "Copy data1", - "type": "Copy", - "dependsOn": [], - "policy": { - "timeout": "7.00:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "userProperties": [], - "typeProperties": { - "source": { - "type": "AzureSqlSource", - "queryTimeout": "02:00:00", - "partitionOption": "None" - }, - "sink": { - "type": "DelimitedTextSink", - "storeSettings": { - "type": "AzureBlobFSWriteSettings" - }, - "formatSettings": { - "type": "DelimitedTextWriteSettings", - "quoteAllText": true, - "fileExtension": ".txt" - } - }, - "enableStaging": false, - "translator": { - "type": "TabularTranslator", - "typeConversion": true, - "typeConversionSettings": { - "allowDataTruncation": true, - "treatBooleanAsNumber": false - } - } - }, - "inputs": [ - { - "referenceName": "ASQL_ProductModel", - "type": "DatasetReference" - } - ], - "outputs": [ - { - "referenceName": "ADLS_Raw", - "type": "DatasetReference" - } - ] - } - ], - "folder": { - "name": "Lab4" - }, - "annotations": [], - "lastPublishTime": "2020-09-26T16:27:34Z" - }, - "type": "Microsoft.DataFactory/factories/pipelines" -} \ No newline at end of file