dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: dennis/Azure-Data-Integration-Pipeline-Training:main
Branches Tags
dennis/Azure-Data-Integration-Pipeline-Training:main dennis/Azure-Data-Integration-Pipeline-Training:one-day-labs-update dennis/Azure-Data-Integration-Pipeline-Training:adf_publish
..
pull from: dennis/Azure-Data-Integration-Pipeline-Training:adf_publish
Branches Tags
dennis/Azure-Data-Integration-Pipeline-Training:main dennis/Azure-Data-Integration-Pipeline-Training:one-day-labs-update dennis/Azure-Data-Integration-Pipeline-Training:adf_publish

10 Commits
main .. adf_publish

Author SHA1 Message Date
Paul Andrew 62020408ab ARM template and parameters deployed on 4-22-2023 16:56:6, based on the collaboration branch's commit ID: 5955c39d4c 2023-04-22 16:56:09 +01:00
Paul Andrew 07d95ecd8a ARM template and parameters deployed on 10-24-2022 11:46:8, based on the collaboration branch's commit ID: 5e460f3eb4 2022-10-24 11:46:11 +01:00
Paul Andrew 38224d36dc ARM template and parameters deployed on 8-30-2022 11:50:9, based on the collaboration branch's commit ID: 08c36321ad 2022-08-30 11:50:13 +01:00
Paul Andrew 2a23b67be2 ARM template and parameters deployed on 3-9-2022 15:59:37, based on the collaboration branch's commit ID: 09c848bc7d 2022-03-09 15:59:39 +00:00
Paul Andrew 104531fb1f ARM template and parameters deployed on 3-1-2022 13:12:12, based on the collaboration branch's commit ID: 2f415a56fc 2022-03-01 13:12:14 +00:00
Paul Andrew 9ef6098405 ARM template and parameters deployed on 2-28-2022 20:15:3, based on the collaboration branch's commit ID: 87fb2ae758 2022-02-28 20:15:04 +00:00
Paul Andrew b7c24b4f99 ARM template and parameters deployed on 2-28-2022 20:10:41, based on the collaboration branch's commit ID: 6cfb707496 2022-02-28 20:10:43 +00:00
Paul Andrew cfb26311e5 ARM template and parameters deployed on 2-28-2022 16:58:47, based on the collaboration branch's commit ID: e6ef95d3cf 2022-02-28 16:58:49 +00:00
Paul Andrew d1c841b473 ARM template and parameters deployed on 2-26-2022 10:40:26, based on the collaboration branch's commit ID: 8d07f3c5d3 2022-02-26 10:40:27 +00:00
Paul Andrew c43f6e5932 Initial commit. 2022-02-26 10:40:00 +00:00
773 changed files with 9307 additions and 27107 deletions
Expand all files Collapse all files
.gitattributes
-2
View File
@@ -1,2 +0,0 @@
# Auto detect text files and perform LF normalization
* text=auto
.github/FUNDING.yml
-3
View File
@@ -1,3 +0,0 @@
# These are supported funding model platforms
github: [mrpaulandrew]
.gitignore
-7
View File
@@ -1,7 +0,0 @@
Code/.vs/TrainingCode/v16/.suo
Code/MetadataDB/trainingdb01.dbmdl
Code/MetadataDB/trainingdb01.jfm
*.dbmdl
*.jfm
Code/.vs/TrainingCode/v16/.suo
Code/.vs/TrainingCode/DesignTimeBuild/.dtbcache.v2
BIN
View File
Binary file not shown.
Code/.vs/TrainingCode/v16/.suo
BIN
View File
Binary file not shown.
Code/Checker/ADF Checker Script v0.1.ps1
-935
View File
@@ -1,935 +0,0 @@
# STEP 1:
#Provide the location of your download ARM template file:
$ARMTemplateFilePath = "C:\ADFRoot\arm_template.json"
#STEP 2:
#Pick how you'd like the output:
[bool]$SummaryOutput = $true
[bool]$VerboseOutput = $false
#STEP 3:
#Run it.
#############################################################################################
if(-not (Test-Path -Path $ARMTemplateFilePath))
{
Write-Error "ARM template file not found. Please check the path provided."
return
}
$Hr = "-------------------------------------------------------------------------------------------------------------------"
Write-Host ""
Write-Host $Hr
Write-Host "Running checks for Data Factory ARM template:"
Write-Host ""
$ARMTemplateFilePath
Write-Host ""
#Parse template into ADF resource parts
$ADF = Get-Content $ARMTemplateFilePath | ConvertFrom-Json
$LinkedServices = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/linkedServices"}
$Datasets = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/datasets"}
$Pipelines = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/pipelines"}
$Activities = $Pipelines.properties.activities #regardless of pipeline
$DataFlows = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/dataflows"}
$Triggers = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/triggers"}
#Output variables
$CheckNumber = 0
$CheckDetail = ""
$Severity = ""
$CheckCounter = 0
$SummaryTable = @()
$VerboseDetailTable = @()
#String helper functions
function CleanName {
param (
[parameter(Mandatory = $true)] [String] $RawValue
)
$CleanName = $RawValue.substring($RawValue.IndexOf("/")+1, $RawValue.LastIndexOf("'") - $RawValue.IndexOf("/")-1)
return $CleanName
}
function CleanType {
param (
[parameter(Mandatory = $true)] [String] $RawValue
)
$CleanName = $RawValue.substring($RawValue.LastIndexOf("/")+1, $RawValue.Length - $RawValue.LastIndexOf("/")-1)
return $CleanName
}
#############################################################################################
#Review resource dependants
#############################################################################################
$ResourcesList = New-Object System.Collections.ArrayList($null)
$DependantsList = New-Object System.Collections.ArrayList($null)
#Get resources
ForEach($Resource in $ADF.resources)
{
$ResourceName = CleanName -RawValue $Resource.name
$ResourceType = CleanType -RawValue $Resource.type
$CompleteResource = $ResourceType + "|" + $ResourceName
if(-not ($ResourcesList -contains $CompleteResource))
{
[void]$ResourcesList.Add($CompleteResource)
}
}
#Get dependants
ForEach($Resource in $ADF.resources)# | Where-Object {$_.type -ne "Microsoft.DataFactory/factories/triggers"})
{
if($Resource.dependsOn.Count -eq 1)
{
$DependantName = CleanName -RawValue $Resource.dependsOn[0].ToString()
$CompleteDependant = $DependantName.Replace('/','|')
if(-not ($DependantsList -contains $CompleteDependant))
{
[void]$DependantsList.Add($CompleteDependant)
}
}
else
{
ForEach($Dependant in $Resource.dependsOn)
{
$DependantName = CleanName -RawValue $Dependant
$CompleteDependant = $DependantName.Replace('/','|')
if(-not ($DependantsList -contains $CompleteDependant))
{
[void]$DependantsList.Add($CompleteDependant)
}
}
}
}
#Get trigger dependants
ForEach($Resource in $Triggers)
{
$ResourceName = CleanName -RawValue $Resource.name
$ResourceType = CleanType -RawValue $Resource.type
$CompleteResource = $ResourceType + "|" + $ResourceName
if($Resource.dependsOn.count -ge 1)
{
if(-not ($DependantsList -contains $CompleteResource))
{
[void]$DependantsList.Add($CompleteResource)
}
}
}
#Establish simple redundancy to use later
$RedundantResources = $ResourcesList | Where-Object {$DependantsList -notcontains $_}
#############################################################################################
#Check for pipeline without triggers
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) without any triggers attached. Directly or indirectly."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "pipelines*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $Parts[1];
CheckDetail = "Does not any triggers attached.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check pipeline with an impossible execution chain.
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) with an impossible AND/OR activity execution chain."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
ForEach($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$ActivityFailureDependencies = New-Object System.Collections.ArrayList($null)
$ActivitySuccessDependencies = New-Object System.Collections.ArrayList($null)
#get upstream failure dependants
ForEach($Activity in $Pipeline.properties.activities)
{
if($Activity.dependsOn.Count -gt 1)
{
ForEach($UpStreamActivity in $Activity.dependsOn)
{
if($UpStreamActivity.dependencyConditions.Contains('Failed'))
{
if(-not ($ActivityFailureDependencies -contains $UpStreamActivity.activity))
{
[void]$ActivityFailureDependencies.Add($UpStreamActivity.activity)
}
}
}
}
}
#get downstream success dependants
ForEach($ActivityDependant in $ActivityFailureDependencies)
{
ForEach($Activity in $Pipeline.properties.activities | Where-Object {$_.name -eq $ActivityDependant})
{
if($Activity.dependsOn.Count -ge 1)
{
ForEach($DownStreamActivity in $Activity.dependsOn)
{
if($DownStreamActivity.dependencyConditions.Contains('Succeeded'))
{
if(-not ($ActivitySuccessDependencies -contains $DownStreamActivity.activity))
{
[void]$ActivitySuccessDependencies.Add($DownStreamActivity.activity)
}
}
}
}
}
}
#compare dependants - do they exist in both lists?
$Problems = $ActivityFailureDependencies | Where-Object {$ActivitySuccessDependencies -contains $_}
if($Problems.Count -gt 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Has an impossible AND/OR activity execution chain.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for pipeline descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$PipelineDescription = $Pipeline.properties.description
if(([string]::IsNullOrEmpty($PipelineDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for pipelines not in folders
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) not organised into folders."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$PipelineFolder = $Pipeline.properties.folder.name
if(([string]::IsNullOrEmpty($PipelineFolder)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Not organised into a folder.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for pipelines without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$PipelineAnnotations = $Pipeline.properties.annotations.Count
if($PipelineAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for data flow descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Data Flow(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($DataFlow in $DataFlows)
{
$DataFlowName = (CleanName -RawValue $DataFlow.name.ToString())
$DataFlowDescription = $DataFlow.properties.description
if(([string]::IsNullOrEmpty($DataFlowDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Data Flow";
Name = $DataFlowName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check activity timeout values
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) with timeout values still set to the service default value of 7 days."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
ForEach ($Activity in $Activities)
{
$timeout = $Activity.policy.timeout
if(-not ([string]::IsNullOrEmpty($timeout)))
{
if($timeout -eq "7.00:00:00")
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "Timeout policy still set to the service default value of 7 days.";
Severity = $Severity
}
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check activity descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Activity in $Activities)
{
$ActivityDescription = $Activity.description
if(([string]::IsNullOrEmpty($ActivityDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check foreach activity batch size unset
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) ForEach iteration without a batch count value set."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
{
[bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
{
$isSequential = $Activity.typeProperties.isSequential
}
$BatchCount = $Activity.typeProperties.batchCount
if(!$isSequential)
{
if(([string]::IsNullOrEmpty($BatchCount)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "ForEach does not have a batch count value set.";
Severity = $Severity
}
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check foreach activity batch size is less than the service maximum
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) ForEach iteration with a batch count size that is less than the service maximum."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
{
[bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
{
$isSequential = $Activity.typeProperties.isSequential
}
$BatchCount = $Activity.typeProperties.batchCount
if(!$isSequential)
{
if($BatchCount -lt 50)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "ForEach has a batch size that is less than the service maximum.";
Severity = $Severity
}
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check linked service using key vault
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) not using Azure Key Vault to store credentials."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
$LinkedServiceList = New-Object System.Collections.ArrayList($null)
ForEach ($LinkedService in $LinkedServices | Where-Object {$_.properties.type -ne "AzureKeyVault"})
{
$typeProperties = Get-Member -InputObject $LinkedService.properties.typeProperties -MemberType NoteProperty
ForEach($typeProperty in $typeProperties)
{
$propValue = $LinkedService.properties.typeProperties | Select-Object -ExpandProperty $typeProperty.Name
#handle linked services with multiple type properties
if(([string]::IsNullOrEmpty($propValue.secretName))){
$LinkedServiceName = (CleanName -RawValue $LinkedService.name)
if(-not ($LinkedServiceList -contains $LinkedServiceName))
{
[void]$LinkedServiceList.Add($LinkedServiceName) #add linked service if secretName is missing
}
}
if(-not([string]::IsNullOrEmpty($propValue.secretName))){
$LinkedServiceName = (CleanName -RawValue $LinkedService.name)
[void]$LinkedServiceList.Remove($LinkedServiceName) #renove linked service if secretName is then found
}
}
}
$CheckCounter = $LinkedServiceList.Count
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
if($VerboseOutput)
{
ForEach ($LinkedServiceOutput in $LinkedServiceList)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $LinkedServiceOutput;
CheckDetail = "Not using Key Vault to store credentials.";
Severity = $Severity
}
}
}
#############################################################################################
#Check for linked services not in use
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) not used by any other resource."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "linkedServices*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $Parts[1];
CheckDetail = "Not used by any other resource.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check linked service descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($LinkedService in $LinkedServices)
{
$LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
$LinkedServiceDescription = $LinkedService.properties.description
if(([string]::IsNullOrEmpty($LinkedServiceDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $LinkedServiceName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for linked service without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
$LinkedServiceAnnotations = $Pipeline.properties.annotations.Count
if($LinkedServiceAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $LinkedServiceName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for datasets not in use
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) not used by any other resource."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "datasets*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $Parts[1];
CheckDetail = "Not used by any other resource.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for dataset without description
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Dataset in $Datasets)
{
$DatasetName = (CleanName -RawValue $Dataset.name.ToString())
$DatasetDescription = $Dataset.properties.description
if(([string]::IsNullOrEmpty($DatasetDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $DatasetName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check dataset not in folders
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) not organised into folders."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Dataset in $Datasets)
{
$DatasetName = (CleanName -RawValue $Dataset.name.ToString())
$DatasetFolder = $Dataset.properties.folder.name
if(([string]::IsNullOrEmpty($DatasetFolder)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $DatasetName;
CheckDetail = "Not organised into a folder.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for datasets without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Dataset in $Datasets)
{
$DatasetName = (CleanName -RawValue $Dataset.name.ToString())
$DatasetAnnotations = $Dataset.properties.annotations.Count
if($DatasetAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $DatasetName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for triggers not in use
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Trigger(s) not used by any other resource."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "triggers*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Trigger";
Name = $Parts[1];
CheckDetail = "Not used by any other resource.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for trigger descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Trigger(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Trigger in $Triggers)
{
$TriggerName = (CleanName -RawValue $Pipeline.name.ToString())
$TriggerDescription = $Trigger.properties.description
if(([string]::IsNullOrEmpty($TriggerDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Trigger";
Name = $TriggerName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for trigger without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Trigger(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Trigger in $Triggers)
{
$TriggerName = (CleanName -RawValue $Trigger.name.ToString())
$TriggerAnnotations = $Trigger.properties.annotations.Count
if($TriggerAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Trigger";
Name = $TriggerName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
Write-Host ""
Write-Host $Hr
if($SummaryOutput)
{
Write-Host ""
Write-Host "Results Summary:"
Write-Host ""
Write-Host "Checks ran against template:" $CheckNumber
Write-Host "Checks with issues found:" ($SummaryTable | Where-Object {$_.IssueCount -ne 0}).Count.ToString()
Write-Host "Total issue count:" ($SummaryTable | Measure-Object -Property IssueCount -Sum).Sum
$SummaryTable | Where-Object {$_.IssueCount -ne 0} | Format-Table @{
Label = "Issue Count";Expression = {$_.IssueCount}; Alignment="Center"}, @{
Label = "Check Details";Expression = {$_.CheckDetail}}, @{
Label = "Severity"
Expression =
{
switch ($_.Severity)
{
#https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
'Low' {$color = "92"; break }
'Medium' {$color = '93'; break }
'High' {$color = "31"; break }
default {$color = "0"}
}
$e = [char]27
"$e[${color}m$($_.Severity)${e}[0m"
}
}
Write-Host $Hr
}
if($VerboseOutput)
{
Write-Host ""
Write-Host "Results Details:"
$VerboseDetailTable | Format-Table @{
Label = "Component";Expression = {$_.Component}}, @{
Label = "Name";Expression = {$_.Name}}, @{
Label = "Check Detail";Expression = {$_.CheckDetail}}, @{
Label = "Severity"
Expression =
{
switch ($_.Severity)
{
#https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
'Low' {$color = "92"; break }
'Medium' {$color = '93'; break }
'High' {$color = "31"; break }
default {$color = "0"}
}
$e = [char]27
"$e[${color}m$($_.Severity)${e}[0m"
}
}
Write-Host $Hr
}
Code/CustomActivity/CustomActivity.csproj
-8
View File
@@ -1,8 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
</Project>
Code/CustomActivity/Program.cs
-15
View File
@@ -1,15 +0,0 @@
using System;
using System.Threading;
namespace CustomActivity
{
class Program
{
static void Main(string[] args)
{
Console.WriteLine("Hello World!");
Thread.Sleep(120000); // sleep 3mins
Console.WriteLine("Done");
}
}
}
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.deps.json
-23
View File
@@ -1,23 +0,0 @@
{
"runtimeTarget": {
"name": ".NETCoreApp,Version=v3.1",
"signature": ""
},
"compilationOptions": {},
"targets": {
".NETCoreApp,Version=v3.1": {
"CustomActivity/1.0.0": {
"runtime": {
"CustomActivity.dll": {}
}
}
}
},
"libraries": {
"CustomActivity/1.0.0": {
"type": "project",
"serviceable": false,
"sha512": ""
}
}
}
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.dll
BIN
View File
Binary file not shown.
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.exe
BIN
View File
Binary file not shown.
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.pdb
BIN
View File
Binary file not shown.
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.runtimeconfig.dev.json
-9
View File
@@ -1,9 +0,0 @@
{
"runtimeOptions": {
"additionalProbingPaths": [
"C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
"C:\\Users\\paul.andrew\\.nuget\\packages",
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
]
}
}
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.runtimeconfig.json
-9
View File
@@ -1,9 +0,0 @@
{
"runtimeOptions": {
"tfm": "netcoreapp3.1",
"framework": {
"name": "Microsoft.NETCore.App",
"version": "3.1.0"
}
}
}
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.deps.json
-23
View File
@@ -1,23 +0,0 @@
{
"runtimeTarget": {
"name": ".NETCoreApp,Version=v3.1",
"signature": ""
},
"compilationOptions": {},
"targets": {
".NETCoreApp,Version=v3.1": {
"CustomActivity/1.0.0": {
"runtime": {
"CustomActivity.dll": {}
}
}
}
},
"libraries": {
"CustomActivity/1.0.0": {
"type": "project",
"serviceable": false,
"sha512": ""
}
}
}
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.dll
BIN
View File
Binary file not shown.
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.exe
BIN
View File
Binary file not shown.
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.pdb
BIN
View File
Binary file not shown.
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.runtimeconfig.dev.json
-9
View File
@@ -1,9 +0,0 @@
{
"runtimeOptions": {
"additionalProbingPaths": [
"C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
"C:\\Users\\paul.andrew\\.nuget\\packages",
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
]
}
}
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.runtimeconfig.json
-9
View File
@@ -1,9 +0,0 @@
{
"runtimeOptions": {
"tfm": "netcoreapp3.1",
"framework": {
"name": "Microsoft.NETCore.App",
"version": "3.1.0"
}
}
}
Code/CustomActivity/obj/CustomActivity.csproj.nuget.dgspec.json
-66
View File
@@ -1,66 +0,0 @@
{
"format": 1,
"restore": {
"C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {}
},
"projects": {
"C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"projectName": "CustomActivity",
"projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
"outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
"projectStyle": "PackageReference",
"fallbackFolders": [
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
],
"configFilePaths": [
"C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
],
"originalTargetFrameworks": [
"netcoreapp3.1"
],
"sources": {
"C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
}
}
}
}
}
Code/CustomActivity/obj/CustomActivity.csproj.nuget.g.props
-19
View File
@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
<RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
<ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
<NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
<NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\paul.andrew\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
<NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
<NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">5.11.1</NuGetToolVersion>
</PropertyGroup>
<ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<SourceRoot Include="C:\Users\paul.andrew\.nuget\packages\" />
<SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
</ItemGroup>
<PropertyGroup>
<MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
</PropertyGroup>
</Project>
Code/CustomActivity/obj/CustomActivity.csproj.nuget.g.targets
-6
View File
@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
</PropertyGroup>
</Project>
Code/CustomActivity/obj/Debug/netcoreapp3.1/.NETCoreApp,Version=v3.1.AssemblyAttributes.cs
-4
View File
@@ -1,4 +0,0 @@
// <autogenerated />
using System;
using System.Reflection;
[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.AssemblyInfo.cs
-23
View File
@@ -1,23 +0,0 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
using System;
using System.Reflection;
[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
// Generated by the MSBuild WriteCodeFragment class.
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.AssemblyInfoInputs.cache
-1
View File
@@ -1 +0,0 @@
0f25b4827a277d65f1c7ea5fed54274284915748
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
-3
View File
@@ -1,3 +0,0 @@
is_global = true
build_property.RootNamespace = CustomActivity
build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.assets.cache
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.csproj.CoreCompileInputs.cache
-1
View File
@@ -1 +0,0 @@
8c2f34a721f1ab6a8a4b2c0aa34fe087f4b6de2b
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.csproj.FileListAbsolute.txt
-27
View File
@@ -1,27 +0,0 @@
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.AssemblyReference.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.dll
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.genruntimeconfig.cache
-1
View File
@@ -1 +0,0 @@
ed079c0173d4e1bcdf78b18c3f5b795d72f77c29
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.pdb
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Debug/netcoreapp3.1/apphost.exe
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Release/netcoreapp3.1/.NETCoreApp,Version=v3.1.AssemblyAttributes.cs
-4
View File
@@ -1,4 +0,0 @@
// <autogenerated />
using System;
using System.Reflection;
[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.AssemblyInfo.cs
-23
View File
@@ -1,23 +0,0 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
using System;
using System.Reflection;
[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
// Generated by the MSBuild WriteCodeFragment class.
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.AssemblyInfoInputs.cache
-1
View File
@@ -1 +0,0 @@
0f1742c03e055fa8f4a7b3c24bb888fbf0b29ac5
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
-3
View File
@@ -1,3 +0,0 @@
is_global = true
build_property.RootNamespace = CustomActivity
build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.assets.cache
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.csproj.CoreCompileInputs.cache
-1
View File
@@ -1 +0,0 @@
f20592bb3edc01e817172d91bf0bacb295339ec6
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.csproj.FileListAbsolute.txt
-13
View File
@@ -1,13 +0,0 @@
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.exe
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.deps.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.dll
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.genruntimeconfig.cache
-1
View File
@@ -1 +0,0 @@
eca3b3f0408037e0ebe5a5c84e49699310580454
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.pdb
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/Release/netcoreapp3.1/apphost.exe
BIN
View File
Binary file not shown.
Code/CustomActivity/obj/project.assets.json
-72
View File
@@ -1,72 +0,0 @@
{
"version": 3,
"targets": {
".NETCoreApp,Version=v3.1": {}
},
"libraries": {},
"projectFileDependencyGroups": {
".NETCoreApp,Version=v3.1": []
},
"packageFolders": {
"C:\\Users\\paul.andrew\\.nuget\\packages\\": {},
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages": {}
},
"project": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"projectName": "CustomActivity",
"projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
"outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
"projectStyle": "PackageReference",
"fallbackFolders": [
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
],
"configFilePaths": [
"C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
],
"originalTargetFrameworks": [
"netcoreapp3.1"
],
"sources": {
"C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
}
}
}
}
Code/CustomActivity/obj/project.nuget.cache
-8
View File
@@ -1,8 +0,0 @@
{
"version": 2,
"dgSpecHash": "f0p4CwgurPXk2v0yD5sG6nLAvje2rr5wHxFdncruLvsOGiK3Q2I9TnSpBC64Z+fnkevGX0xQTI2XPAbWzSUW3A==",
"success": true,
"projectFilePath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"expectedPackageFiles": [],
"logs": []
}
Code/DataFactory/credential/TrainingCredential01.json
-9
View File
@@ -1,9 +0,0 @@
{
"name": "TrainingCredential01",
"properties": {
"type": "ManagedIdentity",
"typeProperties": {
"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
}
}
}
Code/DataFactory/dataflow/MappingOrderAggregation.json
-42
View File
@@ -1,42 +0,0 @@
{
"name": "MappingOrderAggregation",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "LakeFileOrderHeaderParquet",
"type": "DatasetReference"
},
"name": "OrderHeader"
},
{
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
},
"name": "OrderLineDetails"
}
],
"sinks": [
{
"dataset": {
"referenceName": "TableOrderSummary",
"type": "DatasetReference"
},
"name": "OrderSummary"
}
],
"transformations": [
{
"name": "JoinHeaderToLineDetails"
},
{
"name": "OrderLineCount"
}
],
"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
}
}
}
Code/DataFactory/dataflow/MappingOrderAggregationWithParam.json
-45
View File
@@ -1,45 +0,0 @@
{
"name": "MappingOrderAggregationWithParam",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "LakeFileOrderHeaderParquet",
"type": "DatasetReference"
},
"name": "OrderHeader"
},
{
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
},
"name": "OrderLineDetails"
}
],
"sinks": [
{
"dataset": {
"referenceName": "TableOrderSummary",
"type": "DatasetReference"
},
"name": "OrderSummary"
}
],
"transformations": [
{
"name": "JoinHeaderToLineDetails"
},
{
"name": "OrderLineCount"
},
{
"name": "AddAuditColum"
}
],
"script": "parameters{\n\tAuditColumn as string\n}\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tmatchType:'exact',\n\tignoreSpaces: false,\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount derive(AuditValue = $AuditColumn) ~> AddAuditColum\nAddAuditColum sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
}
}
}
Code/DataFactory/dataflow/UpdateProductDimension.json
-165
View File
@@ -1,165 +0,0 @@
{
"name": "UpdateProductDimension",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}
Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json
-169
View File
@@ -1,169 +0,0 @@
{
"name": "UpdateProductDimension_Sorted",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
},
{
"name": "SortBySubcategory"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
"SortBySubcategory sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}
Code/DataFactory/dataflow/WranglingOrderAggregation.json
-19
View File
@@ -1,19 +0,0 @@
{
"name": "WranglingOrderAggregation",
"properties": {
"type": "WranglingDataFlow",
"typeProperties": {
"sources": [
{
"name": "LakeFileOrderDetailLinesParquet",
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
}
}
],
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
}
}
}
Code/DataFactory/dataset/ADLS_BIN_AWProduct.json
-22
View File
@@ -1,22 +0,0 @@
{
"name": "ADLS_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Product.csv",
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}
Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json
-30
View File
@@ -1,30 +0,0 @@
{
"name": "ADLS_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().Entity}.tsv",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}
Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json
-32
View File
@@ -1,32 +0,0 @@
{
"name": "ADLS_PQT_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"EntityName": {
"type": "string"
}
},
"folder": {
"name": "Labs2"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().EntityName}.parquet",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"compressionCodec": "snappy"
},
"schema": []
}
}
Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json
-34
View File
@@ -1,34 +0,0 @@
{
"name": "ADLS_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"FileName": {
"type": "string"
}
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().FileName",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
}
}
Code/DataFactory/dataset/AnyDatabaseTable.json
-43
View File
@@ -1,43 +0,0 @@
{
"name": "AnyDatabaseTable",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnection",
"type": "LinkedServiceReference",
"parameters": {
"DBConnectionSecret": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/AnyDatabaseTableAnyKeyVault.json
-50
View File
@@ -1,50 +0,0 @@
{
"name": "AnyDatabaseTableAnyKeyVault",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
"type": "LinkedServiceReference",
"parameters": {
"KeyVaultURL": {
"value": "@dataset().KeyVaultURL",
"type": "Expression"
},
"KeyVaultSecretName": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
},
"KeyVaultURL": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/GetSetMetadata.json
-16
View File
@@ -1,16 +0,0 @@
{
"name": "GetSetMetadata",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/HTTP_BIN_AWProduct.json
-19
View File
@@ -1,19 +0,0 @@
{
"name": "HTTP_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWProduct",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation"
}
}
}
}
Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json
-28
View File
@@ -1,28 +0,0 @@
{
"name": "HTTP_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().Entity}.csv",
"type": "Expression"
}
}
}
}
}
Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json
-32
View File
@@ -1,32 +0,0 @@
{
"name": "HTTP_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"EntityName": {
"type": "string"
}
},
"folder": {
"name": "Labs2"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().EntityName}.csv",
"type": "Expression"
}
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
}
}
Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json
-68
View File
@@ -1,68 +0,0 @@
{
"name": "LakeFileOrderDetailLinesParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderDetailLines.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "SalesOrderDetailID",
"type": "INT32"
},
{
"name": "OrderQty",
"type": "INT32"
},
{
"name": "ProductID",
"type": "INT32"
},
{
"name": "UnitPrice",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "UnitPriceDiscount",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "LineTotal",
"type": "DECIMAL",
"precision": 38,
"scale": 6
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json
-122
View File
@@ -1,122 +0,0 @@
{
"name": "LakeFileOrderHeaderParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderHeader.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "RevisionNumber",
"type": "INT32"
},
{
"name": "OrderDate",
"type": "INT96"
},
{
"name": "DueDate",
"type": "INT96"
},
{
"name": "ShipDate",
"type": "INT96"
},
{
"name": "Status",
"type": "INT32"
},
{
"name": "OnlineOrderFlag",
"type": "BOOLEAN"
},
{
"name": "SalesOrderNumber",
"type": "UTF8"
},
{
"name": "PurchaseOrderNumber",
"type": "UTF8"
},
{
"name": "AccountNumber",
"type": "UTF8"
},
{
"name": "CustomerID",
"type": "INT32"
},
{
"name": "ShipToAddressID",
"type": "INT32"
},
{
"name": "BillToAddressID",
"type": "INT32"
},
{
"name": "ShipMethod",
"type": "UTF8"
},
{
"name": "CreditCardApprovalCode",
"type": "UTF8"
},
{
"name": "SubTotal",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TaxAmt",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Freight",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TotalDue",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Comment",
"type": "UTF8"
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LakeFileParquet.json
-39
View File
@@ -1,39 +0,0 @@
{
"name": "LakeFileParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LakeFilePersonCSV.json
-37
View File
@@ -1,37 +0,0 @@
{
"name": "LakeFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Person.csv",
"folderPath": "Landing",
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LakeFiles.json
-41
View File
@@ -1,41 +0,0 @@
{
"name": "LakeFiles",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LakeFilesBinary.json
-36
View File
@@ -1,36 +0,0 @@
{
"name": "LakeFilesBinary",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
}
}
}
}
Code/DataFactory/dataset/LaptopFilePersonCSV.json
-36
View File
@@ -1,36 +0,0 @@
{
"name": "LaptopFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "Person.csv",
"folderPath": "ForUpload/People"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LaptopFilesBinary.json
-35
View File
@@ -1,35 +0,0 @@
{
"name": "LaptopFilesBinary",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
}
}
}
}
Code/DataFactory/dataset/LaptopFilesParquet.json
-37
View File
@@ -1,37 +0,0 @@
{
"name": "LaptopFilesParquet",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"compressionCodec": "snappy"
},
"schema": []
}
}
Code/DataFactory/dataset/LaptopFolders.json
-33
View File
@@ -1,33 +0,0 @@
{
"name": "LaptopFolders",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/LaptopsFiles.json
-40
View File
@@ -1,40 +0,0 @@
{
"name": "LaptopsFiles",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/dataset/TableOrderSummary.json
-30
View File
@@ -1,30 +0,0 @@
{
"name": "TableOrderSummary",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "SalesOrderNumber",
"type": "varchar"
},
{
"name": "RecordCount",
"type": "int",
"precision": 10
}
],
"typeProperties": {
"schema": "dbo",
"table": "OrderSummary"
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}
Code/DataFactory/factory/TrainingFactoryDev.json
-30
View File
@@ -1,30 +0,0 @@
{
"name": "TrainingFactoryDev",
"properties": {
"globalParameters": {
"Environment": {
"type": "string",
"value": "Dev"
},
"KeyVaultName": {
"type": "string",
"value": "trainingkeys01"
}
},
"globalConfigurations": {
"PipelineBillingEnabled": "true"
}
},
"location": "uksouth",
"identity": {
"type": "SystemAssigned,UserAssigned",
"principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f",
"tenantId": "929039b9-c19d-425e-993b-1e3fda155876",
"userAssignedIdentities": {
"/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": {
"clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85",
"principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91"
}
}
}
}
Code/DataFactory/factory/adayfullofadf-adf.json
-4
View File
@@ -1,4 +0,0 @@
{
"name": "adayfullofadf-adf",
"location": "uksouth"
}
Code/DataFactory/integrationRuntime/AzureVM.json
-6
View File
@@ -1,6 +0,0 @@
{
"name": "AzureVM",
"properties": {
"type": "SelfHosted"
}
}
Code/DataFactory/integrationRuntime/ForDataFlowDemos.json
-16
View File
@@ -1,16 +0,0 @@
{
"name": "ForDataFlowDemos",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 240
}
}
}
}
}
Code/DataFactory/integrationRuntime/Laptop.json
-6
View File
@@ -1,6 +0,0 @@
{
"name": "Laptop",
"properties": {
"type": "SelfHosted"
}
}
Code/DataFactory/integrationRuntime/VNetEnabledIR.json
-21
View File
@@ -1,21 +0,0 @@
{
"name": "VNetEnabledIR",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 0,
"cleanup": true
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
}
}
Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json
-18
View File
@@ -1,18 +0,0 @@
{
"name": "ADLS_saintegrationpipelines",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
}
}
Code/DataFactory/linkedService/AnyDatabaseConnection.json
-26
View File
@@ -1,26 +0,0 @@
{
"name": "AnyDatabaseConnection",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"parameters": {
"DBConnectionSecret": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": {
"value": "@linkedService().DBConnectionSecret",
"type": "Expression"
}
}
}
}
}
Code/DataFactory/linkedService/AnyDatabaseConnectionFromAnyKeyVault.json
-34
View File
@@ -1,34 +0,0 @@
{
"name": "AnyDatabaseConnectionFromAnyKeyVault",
"properties": {
"parameters": {
"KeyVaultURL": {
"type": "string"
},
"KeyVaultSecretName": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "GenericKeys",
"type": "LinkedServiceReference",
"parameters": {
"baseUrl": {
"value": "@linkedService().KeyVaultURL",
"type": "Expression"
}
}
},
"secretName": {
"value": "@linkedService().KeyVaultSecretName",
"type": "Expression"
}
}
}
}
}
Code/DataFactory/linkedService/AzureDataLakeStorage1.json
-11
View File
@@ -1,11 +0,0 @@
{
"name": "AzureDataLakeStorage1",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://adayfullofadfsa.dfs.core.windows.net"
}
}
}
Code/DataFactory/linkedService/AzureSqlDatabase1.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "AzureSqlDatabase1",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=adayfullofadf-sql.database.windows.net;initial catalog=AdventureWorks;user id=sql-admin",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFEQVlGVUxMT0ZBREYtQURGXzA3MGU1MDIyLWEzYjMtNDFmMS1hOTk3LWU2ODE3Y2VhNjdhYyINCn0="
}
}
}
Code/DataFactory/linkedService/BatchForTraining01.json
-17
View File
@@ -1,17 +0,0 @@
{
"name": "BatchForTraining01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBatch",
"typeProperties": {
"batchUri": "https://batchfortraining01.uksouth.batch.azure.com",
"poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654",
"accountName": "batchfortraining01",
"linkedServiceName": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
}
}
}
}
Code/DataFactory/linkedService/EmailSenderFunction.json
-13
View File
@@ -1,13 +0,0 @@
{
"name": "EmailSenderFunction",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureFunction",
"typeProperties": {
"functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net",
"authentication": "Anonymous",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9jZGYyMmI4Ni0zNTk2LTRkNzktYjdmOC1kNGNiMjUyNDZmYjUiDQp9"
}
}
}
Code/DataFactory/linkedService/GenericKeys.json
-16
View File
@@ -1,16 +0,0 @@
{
"name": "GenericKeys",
"properties": {
"type": "AzureKeyVault",
"parameters": {
"baseUrl": {
"type": "String"
}
},
"annotations": [],
"typeProperties": {
"baseUrl": "@{linkedService().baseUrl}"
},
"description": "https://TrainingKeys01.vault.azure.net"
}
}
Code/DataFactory/linkedService/HTTP_AWGitHub.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "HTTP_AWGitHub",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}
Code/DataFactory/linkedService/HTTP_AWProduct.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "HTTP_AWProduct",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}
Code/DataFactory/linkedService/LaptopFiles.json
-24
View File
@@ -1,24 +0,0 @@
{
"name": "LaptopFiles",
"properties": {
"annotations": [],
"type": "FileServer",
"typeProperties": {
"host": "C:\\ADFRoot\\",
"userId": "ADFIRUser",
"password": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ADFIRUser"
}
},
"connectVia": {
"referenceName": "Laptop",
"type": "IntegrationRuntimeReference"
}
},
"type": "Microsoft.DataFactory/factories/linkedservices"
}
Code/DataFactory/linkedService/TrainingKeys01.json
-11
View File
@@ -1,11 +0,0 @@
{
"name": "TrainingKeys01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "https://TrainingKeys01.vault.azure.net/"
}
}
}
Code/DataFactory/linkedService/TrainingKeys01withUMI.json
-15
View File
@@ -1,15 +0,0 @@
{
"name": "TrainingKeys01withUMI",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "https://TrainingKeys01.vault.azure.net/",
"credential": {
"referenceName": "TrainingCredential01",
"type": "CredentialReference"
}
}
}
}
Code/DataFactory/linkedService/TrainingStore01.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "TrainingStore01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobStorage",
"typeProperties": {
"connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9"
}
}
}
Code/DataFactory/linkedService/trainingdb01.json
-18
View File
@@ -1,18 +0,0 @@
{
"name": "trainingdb01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ConnectionString-trainingdb01"
}
}
}
}
Code/DataFactory/linkedService/traininglak01.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "traininglak01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9"
}
}
}
Code/DataFactory/linkedService/traininglake01.json
-19
View File
@@ -1,19 +0,0 @@
{
"name": "traininglake01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
}
}
Code/DataFactory/linkedService/traininglake01noneKV.json
-12
View File
@@ -1,12 +0,0 @@
{
"name": "traininglake01noneKV",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9"
}
}
}
Code/DataFactory/managedVirtualNetwork/default.json
-4
View File
@@ -1,4 +0,0 @@
{
"name": "default",
"type": "Microsoft.DataFactory/factories/managedvirtualnetworks"
}

Some files were not shown because too many files have changed in this diff Show More