dennis/Azure-Data-Integration-Pipeline-Training
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: dennis:main
Branches Tags
dennis:main
dennis:one-day-labs-update
dennis:adf_publish
...
pull from: dennis:adf_publish
Branches Tags
dennis:main
dennis:one-day-labs-update
dennis:adf_publish

No commits in common. "main" and "adf_publish" have entirely different histories.
main ... adf_publis

773 changed files with 9307 additions and 27107 deletions
Show Stats Expand all files Collapse all files

2
.gitattributes vendored
View File

@ -1,2 +0,0 @@
# Auto detect text files and perform LF normalization
* text=auto

3
.github/FUNDING.yml vendored
View File

@ -1,3 +0,0 @@
# These are supported funding model platforms
github: [mrpaulandrew]

7
.gitignore vendored
View File

@ -1,7 +0,0 @@
Code/.vs/TrainingCode/v16/.suo
Code/MetadataDB/trainingdb01.dbmdl
Code/MetadataDB/trainingdb01.jfm
*.dbmdl
*.jfm
Code/.vs/TrainingCode/v16/.suo

BIN
Code/.vs/TrainingCode/DesignTimeBuild/.dtbcache.v2
View File

Binary file not shown.

BIN
Code/.vs/TrainingCode/v16/.suo
View File

Binary file not shown.

935
Code/Checker/ADF Checker Script v0.1.ps1
View File

@ -1,935 +0,0 @@
# STEP 1:
#Provide the location of your download ARM template file:
$ARMTemplateFilePath = "C:\ADFRoot\arm_template.json"
#STEP 2:
#Pick how you'd like the output:
[bool]$SummaryOutput = $true
[bool]$VerboseOutput = $false
#STEP 3:
#Run it.
#############################################################################################
if(-not (Test-Path -Path $ARMTemplateFilePath))
{
Write-Error "ARM template file not found. Please check the path provided."
return
}
$Hr = "-------------------------------------------------------------------------------------------------------------------"
Write-Host ""
Write-Host $Hr
Write-Host "Running checks for Data Factory ARM template:"
Write-Host ""
$ARMTemplateFilePath
Write-Host ""
#Parse template into ADF resource parts
$ADF = Get-Content $ARMTemplateFilePath | ConvertFrom-Json
$LinkedServices = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/linkedServices"}
$Datasets = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/datasets"}
$Pipelines = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/pipelines"}
$Activities = $Pipelines.properties.activities #regardless of pipeline
$DataFlows = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/dataflows"}
$Triggers = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/triggers"}
#Output variables
$CheckNumber = 0
$CheckDetail = ""
$Severity = ""
$CheckCounter = 0
$SummaryTable = @()
$VerboseDetailTable = @()
#String helper functions
function CleanName {
param (
[parameter(Mandatory = $true)] [String] $RawValue
)
$CleanName = $RawValue.substring($RawValue.IndexOf("/")+1, $RawValue.LastIndexOf("'") - $RawValue.IndexOf("/")-1)
return $CleanName
}
function CleanType {
param (
[parameter(Mandatory = $true)] [String] $RawValue
)
$CleanName = $RawValue.substring($RawValue.LastIndexOf("/")+1, $RawValue.Length - $RawValue.LastIndexOf("/")-1)
return $CleanName
}
#############################################################################################
#Review resource dependants
#############################################################################################
$ResourcesList = New-Object System.Collections.ArrayList($null)
$DependantsList = New-Object System.Collections.ArrayList($null)
#Get resources
ForEach($Resource in $ADF.resources)
{
$ResourceName = CleanName -RawValue $Resource.name
$ResourceType = CleanType -RawValue $Resource.type
$CompleteResource = $ResourceType + "|" + $ResourceName
if(-not ($ResourcesList -contains $CompleteResource))
{
[void]$ResourcesList.Add($CompleteResource)
}
}
#Get dependants
ForEach($Resource in $ADF.resources)# | Where-Object {$_.type -ne "Microsoft.DataFactory/factories/triggers"})
{
if($Resource.dependsOn.Count -eq 1)
{
$DependantName = CleanName -RawValue $Resource.dependsOn[0].ToString()
$CompleteDependant = $DependantName.Replace('/','|')
if(-not ($DependantsList -contains $CompleteDependant))
{
[void]$DependantsList.Add($CompleteDependant)
}
}
else
{
ForEach($Dependant in $Resource.dependsOn)
{
$DependantName = CleanName -RawValue $Dependant
$CompleteDependant = $DependantName.Replace('/','|')
if(-not ($DependantsList -contains $CompleteDependant))
{
[void]$DependantsList.Add($CompleteDependant)
}
}
}
}
#Get trigger dependants
ForEach($Resource in $Triggers)
{
$ResourceName = CleanName -RawValue $Resource.name
$ResourceType = CleanType -RawValue $Resource.type
$CompleteResource = $ResourceType + "|" + $ResourceName
if($Resource.dependsOn.count -ge 1)
{
if(-not ($DependantsList -contains $CompleteResource))
{
[void]$DependantsList.Add($CompleteResource)
}
}
}
#Establish simple redundancy to use later
$RedundantResources = $ResourcesList | Where-Object {$DependantsList -notcontains $_}
#############################################################################################
#Check for pipeline without triggers
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) without any triggers attached. Directly or indirectly."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "pipelines*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $Parts[1];
CheckDetail = "Does not any triggers attached.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check pipeline with an impossible execution chain.
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) with an impossible AND/OR activity execution chain."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
ForEach($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$ActivityFailureDependencies = New-Object System.Collections.ArrayList($null)
$ActivitySuccessDependencies = New-Object System.Collections.ArrayList($null)
#get upstream failure dependants
ForEach($Activity in $Pipeline.properties.activities)
{
if($Activity.dependsOn.Count -gt 1)
{
ForEach($UpStreamActivity in $Activity.dependsOn)
{
if($UpStreamActivity.dependencyConditions.Contains('Failed'))
{
if(-not ($ActivityFailureDependencies -contains $UpStreamActivity.activity))
{
[void]$ActivityFailureDependencies.Add($UpStreamActivity.activity)
}
}
}
}
}
#get downstream success dependants
ForEach($ActivityDependant in $ActivityFailureDependencies)
{
ForEach($Activity in $Pipeline.properties.activities | Where-Object {$_.name -eq $ActivityDependant})
{
if($Activity.dependsOn.Count -ge 1)
{
ForEach($DownStreamActivity in $Activity.dependsOn)
{
if($DownStreamActivity.dependencyConditions.Contains('Succeeded'))
{
if(-not ($ActivitySuccessDependencies -contains $DownStreamActivity.activity))
{
[void]$ActivitySuccessDependencies.Add($DownStreamActivity.activity)
}
}
}
}
}
}
#compare dependants - do they exist in both lists?
$Problems = $ActivityFailureDependencies | Where-Object {$ActivitySuccessDependencies -contains $_}
if($Problems.Count -gt 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Has an impossible AND/OR activity execution chain.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for pipeline descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$PipelineDescription = $Pipeline.properties.description
if(([string]::IsNullOrEmpty($PipelineDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for pipelines not in folders
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) not organised into folders."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$PipelineFolder = $Pipeline.properties.folder.name
if(([string]::IsNullOrEmpty($PipelineFolder)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Not organised into a folder.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for pipelines without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Pipeline(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
$PipelineAnnotations = $Pipeline.properties.annotations.Count
if($PipelineAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Pipeline";
Name = $PipelineName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for data flow descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Data Flow(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($DataFlow in $DataFlows)
{
$DataFlowName = (CleanName -RawValue $DataFlow.name.ToString())
$DataFlowDescription = $DataFlow.properties.description
if(([string]::IsNullOrEmpty($DataFlowDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Data Flow";
Name = $DataFlowName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check activity timeout values
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) with timeout values still set to the service default value of 7 days."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
ForEach ($Activity in $Activities)
{
$timeout = $Activity.policy.timeout
if(-not ([string]::IsNullOrEmpty($timeout)))
{
if($timeout -eq "7.00:00:00")
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "Timeout policy still set to the service default value of 7 days.";
Severity = $Severity
}
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check activity descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Activity in $Activities)
{
$ActivityDescription = $Activity.description
if(([string]::IsNullOrEmpty($ActivityDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check foreach activity batch size unset
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) ForEach iteration without a batch count value set."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
{
[bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
{
$isSequential = $Activity.typeProperties.isSequential
}
$BatchCount = $Activity.typeProperties.batchCount
if(!$isSequential)
{
if(([string]::IsNullOrEmpty($BatchCount)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "ForEach does not have a batch count value set.";
Severity = $Severity
}
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check foreach activity batch size is less than the service maximum
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Activitie(s) ForEach iteration with a batch count size that is less than the service maximum."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
{
[bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
{
$isSequential = $Activity.typeProperties.isSequential
}
$BatchCount = $Activity.typeProperties.batchCount
if(!$isSequential)
{
if($BatchCount -lt 50)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Activity";
Name = $Activity.Name;
CheckDetail = "ForEach has a batch size that is less than the service maximum.";
Severity = $Severity
}
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check linked service using key vault
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) not using Azure Key Vault to store credentials."
Write-Host "Running check... " $CheckDetail
$Severity = "High"
$LinkedServiceList = New-Object System.Collections.ArrayList($null)
ForEach ($LinkedService in $LinkedServices | Where-Object {$_.properties.type -ne "AzureKeyVault"})
{
$typeProperties = Get-Member -InputObject $LinkedService.properties.typeProperties -MemberType NoteProperty
ForEach($typeProperty in $typeProperties)
{
$propValue = $LinkedService.properties.typeProperties | Select-Object -ExpandProperty $typeProperty.Name
#handle linked services with multiple type properties
if(([string]::IsNullOrEmpty($propValue.secretName))){
$LinkedServiceName = (CleanName -RawValue $LinkedService.name)
if(-not ($LinkedServiceList -contains $LinkedServiceName))
{
[void]$LinkedServiceList.Add($LinkedServiceName) #add linked service if secretName is missing
}
}
if(-not([string]::IsNullOrEmpty($propValue.secretName))){
$LinkedServiceName = (CleanName -RawValue $LinkedService.name)
[void]$LinkedServiceList.Remove($LinkedServiceName) #renove linked service if secretName is then found
}
}
}
$CheckCounter = $LinkedServiceList.Count
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
if($VerboseOutput)
{
ForEach ($LinkedServiceOutput in $LinkedServiceList)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $LinkedServiceOutput;
CheckDetail = "Not using Key Vault to store credentials.";
Severity = $Severity
}
}
}
#############################################################################################
#Check for linked services not in use
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) not used by any other resource."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "linkedServices*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $Parts[1];
CheckDetail = "Not used by any other resource.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check linked service descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($LinkedService in $LinkedServices)
{
$LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
$LinkedServiceDescription = $LinkedService.properties.description
if(([string]::IsNullOrEmpty($LinkedServiceDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $LinkedServiceName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for linked service without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Linked Service(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Pipeline in $Pipelines)
{
$LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
$LinkedServiceAnnotations = $Pipeline.properties.annotations.Count
if($LinkedServiceAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Linked Service";
Name = $LinkedServiceName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for datasets not in use
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) not used by any other resource."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "datasets*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $Parts[1];
CheckDetail = "Not used by any other resource.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for dataset without description
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Dataset in $Datasets)
{
$DatasetName = (CleanName -RawValue $Dataset.name.ToString())
$DatasetDescription = $Dataset.properties.description
if(([string]::IsNullOrEmpty($DatasetDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $DatasetName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check dataset not in folders
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) not organised into folders."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Dataset in $Datasets)
{
$DatasetName = (CleanName -RawValue $Dataset.name.ToString())
$DatasetFolder = $Dataset.properties.folder.name
if(([string]::IsNullOrEmpty($DatasetFolder)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $DatasetName;
CheckDetail = "Not organised into a folder.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for datasets without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Dataset(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Dataset in $Datasets)
{
$DatasetName = (CleanName -RawValue $Dataset.name.ToString())
$DatasetAnnotations = $Dataset.properties.annotations.Count
if($DatasetAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Dataset";
Name = $DatasetName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for triggers not in use
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Trigger(s) not used by any other resource."
Write-Host "Running check... " $CheckDetail
$Severity = "Medium"
ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "triggers*"})
{
$Parts = $RedundantResource.Split('|')
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Trigger";
Name = $Parts[1];
CheckDetail = "Not used by any other resource.";
Severity = $Severity
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for trigger descriptions
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Trigger(s) without a description value."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Trigger in $Triggers)
{
$TriggerName = (CleanName -RawValue $Pipeline.name.ToString())
$TriggerDescription = $Trigger.properties.description
if(([string]::IsNullOrEmpty($TriggerDescription)))
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Trigger";
Name = $TriggerName;
CheckDetail = "Does not have a description.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
#Check for trigger without annotations
#############################################################################################
$CheckNumber += 1
$CheckDetail = "Trigger(s) without annotations."
Write-Host "Running check... " $CheckDetail
$Severity = "Low"
ForEach ($Trigger in $Triggers)
{
$TriggerName = (CleanName -RawValue $Trigger.name.ToString())
$TriggerAnnotations = $Trigger.properties.annotations.Count
if($TriggerAnnotations -le 0)
{
$CheckCounter += 1
if($VerboseOutput)
{
$VerboseDetailTable += [PSCustomObject]@{
Component = "Trigger";
Name = $TriggerName;
CheckDetail = "Does not have any annotations.";
Severity = $Severity
}
}
}
}
$SummaryTable += [PSCustomObject]@{
IssueCount = $CheckCounter;
CheckDetail = $CheckDetail;
Severity = $Severity
}
$CheckCounter = 0
#############################################################################################
Write-Host ""
Write-Host $Hr
if($SummaryOutput)
{
Write-Host ""
Write-Host "Results Summary:"
Write-Host ""
Write-Host "Checks ran against template:" $CheckNumber
Write-Host "Checks with issues found:" ($SummaryTable | Where-Object {$_.IssueCount -ne 0}).Count.ToString()
Write-Host "Total issue count:" ($SummaryTable | Measure-Object -Property IssueCount -Sum).Sum
$SummaryTable | Where-Object {$_.IssueCount -ne 0} | Format-Table @{
Label = "Issue Count";Expression = {$_.IssueCount}; Alignment="Center"}, @{
Label = "Check Details";Expression = {$_.CheckDetail}}, @{
Label = "Severity"
Expression =
{
switch ($_.Severity)
{
#https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
'Low' {$color = "92"; break }
'Medium' {$color = '93'; break }
'High' {$color = "31"; break }
default {$color = "0"}
}
$e = [char]27
"$e[${color}m$($_.Severity)${e}[0m"
}
}
Write-Host $Hr
}
if($VerboseOutput)
{
Write-Host ""
Write-Host "Results Details:"
$VerboseDetailTable | Format-Table @{
Label = "Component";Expression = {$_.Component}}, @{
Label = "Name";Expression = {$_.Name}}, @{
Label = "Check Detail";Expression = {$_.CheckDetail}}, @{
Label = "Severity"
Expression =
{
switch ($_.Severity)
{
#https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
'Low' {$color = "92"; break }
'Medium' {$color = '93'; break }
'High' {$color = "31"; break }
default {$color = "0"}
}
$e = [char]27
"$e[${color}m$($_.Severity)${e}[0m"
}
}
Write-Host $Hr
}

8
Code/CustomActivity/CustomActivity.csproj
View File

@ -1,8 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
</Project>

15
Code/CustomActivity/Program.cs
View File

@ -1,15 +0,0 @@
using System;
using System.Threading;
namespace CustomActivity
{
class Program
{
static void Main(string[] args)
{
Console.WriteLine("Hello World!");
Thread.Sleep(120000); // sleep 3mins
Console.WriteLine("Done");
}
}
}

23
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.deps.json
View File

@ -1,23 +0,0 @@
{
"runtimeTarget": {
"name": ".NETCoreApp,Version=v3.1",
"signature": ""
},
"compilationOptions": {},
"targets": {
".NETCoreApp,Version=v3.1": {
"CustomActivity/1.0.0": {
"runtime": {
"CustomActivity.dll": {}
}
}
}
},
"libraries": {
"CustomActivity/1.0.0": {
"type": "project",
"serviceable": false,
"sha512": ""
}
}
}

BIN
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.dll
View File

Binary file not shown.

BIN
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.exe
View File

Binary file not shown.

BIN
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.pdb
View File

Binary file not shown.

9
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.runtimeconfig.dev.json
View File

@ -1,9 +0,0 @@
{
"runtimeOptions": {
"additionalProbingPaths": [
"C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
"C:\\Users\\paul.andrew\\.nuget\\packages",
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
]
}
}

9
Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.runtimeconfig.json
View File

@ -1,9 +0,0 @@
{
"runtimeOptions": {
"tfm": "netcoreapp3.1",
"framework": {
"name": "Microsoft.NETCore.App",
"version": "3.1.0"
}
}
}

23
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.deps.json
View File

@ -1,23 +0,0 @@
{
"runtimeTarget": {
"name": ".NETCoreApp,Version=v3.1",
"signature": ""
},
"compilationOptions": {},
"targets": {
".NETCoreApp,Version=v3.1": {
"CustomActivity/1.0.0": {
"runtime": {
"CustomActivity.dll": {}
}
}
}
},
"libraries": {
"CustomActivity/1.0.0": {
"type": "project",
"serviceable": false,
"sha512": ""
}
}
}

BIN
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.dll
View File

Binary file not shown.

BIN
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.exe
View File

Binary file not shown.

BIN
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.pdb
View File

Binary file not shown.

9
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.runtimeconfig.dev.json
View File

@ -1,9 +0,0 @@
{
"runtimeOptions": {
"additionalProbingPaths": [
"C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
"C:\\Users\\paul.andrew\\.nuget\\packages",
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
]
}
}

9
Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.runtimeconfig.json
View File

@ -1,9 +0,0 @@
{
"runtimeOptions": {
"tfm": "netcoreapp3.1",
"framework": {
"name": "Microsoft.NETCore.App",
"version": "3.1.0"
}
}
}

66
Code/CustomActivity/obj/CustomActivity.csproj.nuget.dgspec.json
View File

@ -1,66 +0,0 @@
{
"format": 1,
"restore": {
"C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {}
},
"projects": {
"C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"projectName": "CustomActivity",
"projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
"outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
"projectStyle": "PackageReference",
"fallbackFolders": [
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
],
"configFilePaths": [
"C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
],
"originalTargetFrameworks": [
"netcoreapp3.1"
],
"sources": {
"C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
}
}
}
}
}

19
Code/CustomActivity/obj/CustomActivity.csproj.nuget.g.props
View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
<RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
<ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
<NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
<NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\paul.andrew\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
<NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
<NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">5.11.1</NuGetToolVersion>
</PropertyGroup>
<ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<SourceRoot Include="C:\Users\paul.andrew\.nuget\packages\" />
<SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
</ItemGroup>
<PropertyGroup>
<MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
</PropertyGroup>
</Project>

6
Code/CustomActivity/obj/CustomActivity.csproj.nuget.g.targets
View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
</PropertyGroup>
</Project>

4
Code/CustomActivity/obj/Debug/netcoreapp3.1/.NETCoreApp,Version=v3.1.AssemblyAttributes.cs
View File

@ -1,4 +0,0 @@
// <autogenerated />
using System;
using System.Reflection;
[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]

23
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.AssemblyInfo.cs
View File

@ -1,23 +0,0 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
using System;
using System.Reflection;
[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
// Generated by the MSBuild WriteCodeFragment class.

1
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.AssemblyInfoInputs.cache
View File

@ -1 +0,0 @@
0f25b4827a277d65f1c7ea5fed54274284915748

3
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
View File

@ -1,3 +0,0 @@
is_global = true
build_property.RootNamespace = CustomActivity
build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\

BIN
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.assets.cache
View File

Binary file not shown.

1
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.csproj.CoreCompileInputs.cache
View File

@ -1 +0,0 @@
8c2f34a721f1ab6a8a4b2c0aa34fe087f4b6de2b

27
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.csproj.FileListAbsolute.txt
View File

@ -1,27 +0,0 @@
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.AssemblyReference.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache

BIN
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.dll
View File

Binary file not shown.

1
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.genruntimeconfig.cache
View File

@ -1 +0,0 @@
ed079c0173d4e1bcdf78b18c3f5b795d72f77c29

BIN
Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.pdb
View File

Binary file not shown.

BIN
Code/CustomActivity/obj/Debug/netcoreapp3.1/apphost.exe
View File

Binary file not shown.

4
Code/CustomActivity/obj/Release/netcoreapp3.1/.NETCoreApp,Version=v3.1.AssemblyAttributes.cs
View File

@ -1,4 +0,0 @@
// <autogenerated />
using System;
using System.Reflection;
[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]

23
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.AssemblyInfo.cs
View File

@ -1,23 +0,0 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
using System;
using System.Reflection;
[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
// Generated by the MSBuild WriteCodeFragment class.

1
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.AssemblyInfoInputs.cache
View File

@ -1 +0,0 @@
0f1742c03e055fa8f4a7b3c24bb888fbf0b29ac5

3
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
View File

@ -1,3 +0,0 @@
is_global = true
build_property.RootNamespace = CustomActivity
build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\

BIN
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.assets.cache
View File

Binary file not shown.

1
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.csproj.CoreCompileInputs.cache
View File

@ -1 +0,0 @@
f20592bb3edc01e817172d91bf0bacb295339ec6

13
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.csproj.FileListAbsolute.txt
View File

@ -1,13 +0,0 @@
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.exe
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.deps.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.dll
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.pdb
C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.genruntimeconfig.cache

BIN
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.dll
View File

Binary file not shown.

1
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.genruntimeconfig.cache
View File

@ -1 +0,0 @@
eca3b3f0408037e0ebe5a5c84e49699310580454

BIN
Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.pdb
View File

Binary file not shown.

BIN
Code/CustomActivity/obj/Release/netcoreapp3.1/apphost.exe
View File

Binary file not shown.

72
Code/CustomActivity/obj/project.assets.json
View File

@ -1,72 +0,0 @@
{
"version": 3,
"targets": {
".NETCoreApp,Version=v3.1": {}
},
"libraries": {},
"projectFileDependencyGroups": {
".NETCoreApp,Version=v3.1": []
},
"packageFolders": {
"C:\\Users\\paul.andrew\\.nuget\\packages\\": {},
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages": {}
},
"project": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"projectName": "CustomActivity",
"projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
"outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
"projectStyle": "PackageReference",
"fallbackFolders": [
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
],
"configFilePaths": [
"C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
],
"originalTargetFrameworks": [
"netcoreapp3.1"
],
"sources": {
"C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"netcoreapp3.1": {
"targetAlias": "netcoreapp3.1",
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
}
}
}
}

8
Code/CustomActivity/obj/project.nuget.cache
View File

@ -1,8 +0,0 @@
{
"version": 2,
"dgSpecHash": "f0p4CwgurPXk2v0yD5sG6nLAvje2rr5wHxFdncruLvsOGiK3Q2I9TnSpBC64Z+fnkevGX0xQTI2XPAbWzSUW3A==",
"success": true,
"projectFilePath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
"expectedPackageFiles": [],
"logs": []
}

9
Code/DataFactory/credential/TrainingCredential01.json
View File

@ -1,9 +0,0 @@
{
"name": "TrainingCredential01",
"properties": {
"type": "ManagedIdentity",
"typeProperties": {
"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
}
}
}

42
Code/DataFactory/dataflow/MappingOrderAggregation.json
View File

@ -1,42 +0,0 @@
{
"name": "MappingOrderAggregation",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "LakeFileOrderHeaderParquet",
"type": "DatasetReference"
},
"name": "OrderHeader"
},
{
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
},
"name": "OrderLineDetails"
}
],
"sinks": [
{
"dataset": {
"referenceName": "TableOrderSummary",
"type": "DatasetReference"
},
"name": "OrderSummary"
}
],
"transformations": [
{
"name": "JoinHeaderToLineDetails"
},
{
"name": "OrderLineCount"
}
],
"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
}
}
}

45
Code/DataFactory/dataflow/MappingOrderAggregationWithParam.json
View File

@ -1,45 +0,0 @@
{
"name": "MappingOrderAggregationWithParam",
"properties": {
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "LakeFileOrderHeaderParquet",
"type": "DatasetReference"
},
"name": "OrderHeader"
},
{
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
},
"name": "OrderLineDetails"
}
],
"sinks": [
{
"dataset": {
"referenceName": "TableOrderSummary",
"type": "DatasetReference"
},
"name": "OrderSummary"
}
],
"transformations": [
{
"name": "JoinHeaderToLineDetails"
},
{
"name": "OrderLineCount"
},
{
"name": "AddAuditColum"
}
],
"script": "parameters{\n\tAuditColumn as string\n}\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tmatchType:'exact',\n\tignoreSpaces: false,\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount derive(AuditValue = $AuditColumn) ~> AddAuditColum\nAddAuditColum sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
}
}
}

165
Code/DataFactory/dataflow/UpdateProductDimension.json
View File

@ -1,165 +0,0 @@
{
"name": "UpdateProductDimension",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}

169
Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json
View File

@ -1,169 +0,0 @@
{
"name": "UpdateProductDimension_Sorted",
"properties": {
"folder": {
"name": "Labs"
},
"type": "MappingDataFlow",
"typeProperties": {
"sources": [
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "Product"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductSubcategory"
},
{
"dataset": {
"referenceName": "ADLS_TSV_AdventureWorks",
"type": "DatasetReference"
},
"name": "ProductCategory"
}
],
"sinks": [
{
"linkedService": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"name": "WriteToDataLake"
}
],
"transformations": [
{
"name": "SelectProductColumns"
},
{
"name": "SelectSubcategoryColumns"
},
{
"name": "SelectCategoryColumns"
},
{
"name": "LookupProductCategory"
},
{
"name": "LookupProductSubcategory"
},
{
"name": "RemoveDuplicateColumns"
},
{
"name": "SortBySubcategory"
}
],
"scriptLines": [
"source(output(",
" ProductId as integer,",
" Product as string,",
" {_col2_} as string,",
" {_col3_} as boolean,",
" {_col4_} as boolean,",
" {_col5_} as string,",
" {_col6_} as short,",
" {_col7_} as short,",
" {_col8_} as double,",
" {_col9_} as double,",
" {_col10_} as string,",
" {_col11_} as string,",
" {_col12_} as string,",
" {_col13_} as double,",
" {_col14_} as short,",
" {_col15_} as string,",
" {_col16_} as string,",
" {_col17_} as string,",
" SubcategoryId as integer,",
" {_col19_} as short,",
" {_col20_} as timestamp,",
" {_col21_} as timestamp,",
" {_col22_} as string,",
" {_col23_} as string,",
" {_col24_} as string",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> Product",
"source(output(",
" SubcategoryId as integer,",
" CategoryId as integer,",
" Subcategory as string,",
" {_col3_} as string,",
" {_col4_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductSubcategory",
"source(output(",
" CategoryId as integer,",
" Category as string,",
" {_col2_} as string,",
" {_col3_} as timestamp",
" ),",
" allowSchemaDrift: true,",
" validateSchema: false,",
" ignoreNoFilesFound: false) ~> ProductCategory",
"Product select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectProductColumns",
"ProductSubcategory select(mapColumn(",
" SubcategoryId,",
" CategoryId,",
" Subcategory",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
"ProductCategory select(mapColumn(",
" CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductCategory",
"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
" multiple: false,",
" pickup: 'any',",
" broadcast: 'auto')~> LookupProductSubcategory",
"LookupProductSubcategory select(mapColumn(",
" ProductId,",
" Product,",
" SubcategoryId = SelectProductColumns@SubcategoryId,",
" SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
" CategoryId = SelectSubcategoryColumns@CategoryId,",
" Subcategory,",
" CategoryId = SelectCategoryColumns@CategoryId,",
" Category",
" ),",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
"SortBySubcategory sink(allowSchemaDrift: true,",
" validateSchema: false,",
" format: 'parquet',",
" fileSystem: 'lakeroot',",
" folderPath: 'Conformed/DimProduct',",
" truncate: true,",
" umask: 0022,",
" preCommands: [],",
" postCommands: [],",
" skipDuplicateMapInputs: true,",
" skipDuplicateMapOutputs: true) ~> WriteToDataLake"
]
}
}
}

19
Code/DataFactory/dataflow/WranglingOrderAggregation.json
View File

@ -1,19 +0,0 @@
{
"name": "WranglingOrderAggregation",
"properties": {
"type": "WranglingDataFlow",
"typeProperties": {
"sources": [
{
"name": "LakeFileOrderDetailLinesParquet",
"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~> LakeFileOrderDetailLinesParquet",
"dataset": {
"referenceName": "LakeFileOrderDetailLinesParquet",
"type": "DatasetReference"
}
}
],
"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n Parquet = Parquet.Document(AdfDoc)\r\nin\r\n Parquet;\r\nshared UserQuery = let\r\n Source = LakeFileOrderDetailLinesParquet\r\nin\r\n Source;\r\n"
}
}
}

22
Code/DataFactory/dataset/ADLS_BIN_AWProduct.json
View File

@ -1,22 +0,0 @@
{
"name": "ADLS_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Product.csv",
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}

30
Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json
View File

@ -1,30 +0,0 @@
{
"name": "ADLS_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().Entity}.tsv",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
}
}
}
}

32
Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json
View File

@ -1,32 +0,0 @@
{
"name": "ADLS_PQT_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"EntityName": {
"type": "string"
}
},
"folder": {
"name": "Labs2"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@{dataset().EntityName}.parquet",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"compressionCodec": "snappy"
},
"schema": []
}
}

34
Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json
View File

@ -1,34 +0,0 @@
{
"name": "ADLS_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "ADLS_saintegrationpipelines",
"type": "LinkedServiceReference"
},
"parameters": {
"FileName": {
"type": "string"
}
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().FileName",
"type": "Expression"
},
"folderPath": "Raw",
"fileSystem": "lakeroot"
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
}
}

43
Code/DataFactory/dataset/AnyDatabaseTable.json
View File

@ -1,43 +0,0 @@
{
"name": "AnyDatabaseTable",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnection",
"type": "LinkedServiceReference",
"parameters": {
"DBConnectionSecret": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

50
Code/DataFactory/dataset/AnyDatabaseTableAnyKeyVault.json
View File

@ -1,50 +0,0 @@
{
"name": "AnyDatabaseTableAnyKeyVault",
"properties": {
"linkedServiceName": {
"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
"type": "LinkedServiceReference",
"parameters": {
"KeyVaultURL": {
"value": "@dataset().KeyVaultURL",
"type": "Expression"
},
"KeyVaultSecretName": {
"value": "@dataset().LinkedServiceConnectionSecret",
"type": "Expression"
}
}
},
"parameters": {
"LinkedServiceConnectionSecret": {
"type": "string"
},
"SchemaName": {
"type": "string"
},
"TableName": {
"type": "string"
},
"KeyVaultURL": {
"type": "string"
}
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [],
"typeProperties": {
"schema": {
"value": "@dataset().SchemaName",
"type": "Expression"
},
"table": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

16
Code/DataFactory/dataset/GetSetMetadata.json
View File

@ -1,16 +0,0 @@
{
"name": "GetSetMetadata",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

19
Code/DataFactory/dataset/HTTP_BIN_AWProduct.json
View File

@ -1,19 +0,0 @@
{
"name": "HTTP_BIN_AWProduct",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWProduct",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation"
}
}
}
}

28
Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json
View File

@ -1,28 +0,0 @@
{
"name": "HTTP_BIN_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"Entity": {
"type": "string"
}
},
"folder": {
"name": "Labs1"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().Entity}.csv",
"type": "Expression"
}
}
}
}
}

32
Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json
View File

@ -1,32 +0,0 @@
{
"name": "HTTP_TSV_AdventureWorks",
"properties": {
"linkedServiceName": {
"referenceName": "HTTP_AWGitHub",
"type": "LinkedServiceReference"
},
"parameters": {
"EntityName": {
"type": "string"
}
},
"folder": {
"name": "Labs2"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "HttpServerLocation",
"relativeUrl": {
"value": "@{dataset().EntityName}.csv",
"type": "Expression"
}
},
"columnDelimiter": "\t",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
}
}

68
Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json
View File

@ -1,68 +0,0 @@
{
"name": "LakeFileOrderDetailLinesParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderDetailLines.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "SalesOrderDetailID",
"type": "INT32"
},
{
"name": "OrderQty",
"type": "INT32"
},
{
"name": "ProductID",
"type": "INT32"
},
{
"name": "UnitPrice",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "UnitPriceDiscount",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "LineTotal",
"type": "DECIMAL",
"precision": 38,
"scale": 6
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

122
Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json
View File

@ -1,122 +0,0 @@
{
"name": "LakeFileOrderHeaderParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01noneKV",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "OrderHeader.parquet",
"folderPath": "Raw",
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": [
{
"name": "SalesOrderID",
"type": "INT32"
},
{
"name": "RevisionNumber",
"type": "INT32"
},
{
"name": "OrderDate",
"type": "INT96"
},
{
"name": "DueDate",
"type": "INT96"
},
{
"name": "ShipDate",
"type": "INT96"
},
{
"name": "Status",
"type": "INT32"
},
{
"name": "OnlineOrderFlag",
"type": "BOOLEAN"
},
{
"name": "SalesOrderNumber",
"type": "UTF8"
},
{
"name": "PurchaseOrderNumber",
"type": "UTF8"
},
{
"name": "AccountNumber",
"type": "UTF8"
},
{
"name": "CustomerID",
"type": "INT32"
},
{
"name": "ShipToAddressID",
"type": "INT32"
},
{
"name": "BillToAddressID",
"type": "INT32"
},
{
"name": "ShipMethod",
"type": "UTF8"
},
{
"name": "CreditCardApprovalCode",
"type": "UTF8"
},
{
"name": "SubTotal",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TaxAmt",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Freight",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "TotalDue",
"type": "DECIMAL",
"precision": 19,
"scale": 4
},
{
"name": "Comment",
"type": "UTF8"
},
{
"name": "rowguid",
"type": "UTF8"
},
{
"name": "ModifiedDate",
"type": "INT96"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

39
Code/DataFactory/dataset/LakeFileParquet.json
View File

@ -1,39 +0,0 @@
{
"name": "LakeFileParquet",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"compressionCodec": "snappy"
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

37
Code/DataFactory/dataset/LakeFilePersonCSV.json
View File

@ -1,37 +0,0 @@
{
"name": "LakeFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": "Person.csv",
"folderPath": "Landing",
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

41
Code/DataFactory/dataset/LakeFiles.json
View File

@ -1,41 +0,0 @@
{
"name": "LakeFiles",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

36
Code/DataFactory/dataset/LakeFilesBinary.json
View File

@ -1,36 +0,0 @@
{
"name": "LakeFilesBinary",
"properties": {
"linkedServiceName": {
"referenceName": "traininglake01",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Lake"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "AzureBlobFSLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
},
"fileSystem": "datawarehouse"
}
}
}
}

36
Code/DataFactory/dataset/LaptopFilePersonCSV.json
View File

@ -1,36 +0,0 @@
{
"name": "LaptopFilePersonCSV",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": "Person.csv",
"folderPath": "ForUpload/People"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": [
{
"type": "String"
},
{
"type": "String"
},
{
"type": "String"
}
]
},
"type": "Microsoft.DataFactory/factories/datasets"
}

35
Code/DataFactory/dataset/LaptopFilesBinary.json
View File

@ -1,35 +0,0 @@
{
"name": "LaptopFilesBinary",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Binary",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
}
}
}
}

37
Code/DataFactory/dataset/LaptopFilesParquet.json
View File

@ -1,37 +0,0 @@
{
"name": "LaptopFilesParquet",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "Parquet",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"compressionCodec": "snappy"
},
"schema": []
}
}

33
Code/DataFactory/dataset/LaptopFolders.json
View File

@ -1,33 +0,0 @@
{
"name": "LaptopFolders",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

40
Code/DataFactory/dataset/LaptopsFiles.json
View File

@ -1,40 +0,0 @@
{
"name": "LaptopsFiles",
"properties": {
"linkedServiceName": {
"referenceName": "LaptopFiles",
"type": "LinkedServiceReference"
},
"parameters": {
"Directory": {
"type": "string"
},
"File": {
"type": "string"
}
},
"folder": {
"name": "Laptop"
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "FileServerLocation",
"fileName": {
"value": "@dataset().File",
"type": "Expression"
},
"folderPath": {
"value": "@dataset().Directory",
"type": "Expression"
}
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"type": "Microsoft.DataFactory/factories/datasets"
}

30
Code/DataFactory/dataset/TableOrderSummary.json
View File

@ -1,30 +0,0 @@
{
"name": "TableOrderSummary",
"properties": {
"linkedServiceName": {
"referenceName": "trainingdb01",
"type": "LinkedServiceReference"
},
"folder": {
"name": "SQLDB"
},
"annotations": [],
"type": "AzureSqlTable",
"schema": [
{
"name": "SalesOrderNumber",
"type": "varchar"
},
{
"name": "RecordCount",
"type": "int",
"precision": 10
}
],
"typeProperties": {
"schema": "dbo",
"table": "OrderSummary"
}
},
"type": "Microsoft.DataFactory/factories/datasets"
}

30
Code/DataFactory/factory/TrainingFactoryDev.json
View File

@ -1,30 +0,0 @@
{
"name": "TrainingFactoryDev",
"properties": {
"globalParameters": {
"Environment": {
"type": "string",
"value": "Dev"
},
"KeyVaultName": {
"type": "string",
"value": "trainingkeys01"
}
},
"globalConfigurations": {
"PipelineBillingEnabled": "true"
}
},
"location": "uksouth",
"identity": {
"type": "SystemAssigned,UserAssigned",
"principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f",
"tenantId": "929039b9-c19d-425e-993b-1e3fda155876",
"userAssignedIdentities": {
"/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": {
"clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85",
"principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91"
}
}
}
}

4
Code/DataFactory/factory/adayfullofadf-adf.json
View File

@ -1,4 +0,0 @@
{
"name": "adayfullofadf-adf",
"location": "uksouth"
}

6
Code/DataFactory/integrationRuntime/AzureVM.json
View File

@ -1,6 +0,0 @@
{
"name": "AzureVM",
"properties": {
"type": "SelfHosted"
}
}

16
Code/DataFactory/integrationRuntime/ForDataFlowDemos.json
View File

@ -1,16 +0,0 @@
{
"name": "ForDataFlowDemos",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 240
}
}
}
}
}

6
Code/DataFactory/integrationRuntime/Laptop.json
View File

@ -1,6 +0,0 @@
{
"name": "Laptop",
"properties": {
"type": "SelfHosted"
}
}

21
Code/DataFactory/integrationRuntime/VNetEnabledIR.json
View File

@ -1,21 +0,0 @@
{
"name": "VNetEnabledIR",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "UK South",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 0,
"cleanup": true
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
}
}

18
Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json
View File

@ -1,18 +0,0 @@
{
"name": "ADLS_saintegrationpipelines",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
}
}

26
Code/DataFactory/linkedService/AnyDatabaseConnection.json
View File

@ -1,26 +0,0 @@
{
"name": "AnyDatabaseConnection",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"parameters": {
"DBConnectionSecret": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": {
"value": "@linkedService().DBConnectionSecret",
"type": "Expression"
}
}
}
}
}

34
Code/DataFactory/linkedService/AnyDatabaseConnectionFromAnyKeyVault.json
View File

@ -1,34 +0,0 @@
{
"name": "AnyDatabaseConnectionFromAnyKeyVault",
"properties": {
"parameters": {
"KeyVaultURL": {
"type": "string"
},
"KeyVaultSecretName": {
"type": "string"
}
},
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "GenericKeys",
"type": "LinkedServiceReference",
"parameters": {
"baseUrl": {
"value": "@linkedService().KeyVaultURL",
"type": "Expression"
}
}
},
"secretName": {
"value": "@linkedService().KeyVaultSecretName",
"type": "Expression"
}
}
}
}
}

11
Code/DataFactory/linkedService/AzureDataLakeStorage1.json
View File

@ -1,11 +0,0 @@
{
"name": "AzureDataLakeStorage1",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://adayfullofadfsa.dfs.core.windows.net"
}
}
}

12
Code/DataFactory/linkedService/AzureSqlDatabase1.json
View File

@ -1,12 +0,0 @@
{
"name": "AzureSqlDatabase1",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=adayfullofadf-sql.database.windows.net;initial catalog=AdventureWorks;user id=sql-admin",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFEQVlGVUxMT0ZBREYtQURGXzA3MGU1MDIyLWEzYjMtNDFmMS1hOTk3LWU2ODE3Y2VhNjdhYyINCn0="
}
}
}

17
Code/DataFactory/linkedService/BatchForTraining01.json
View File

@ -1,17 +0,0 @@
{
"name": "BatchForTraining01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBatch",
"typeProperties": {
"batchUri": "https://batchfortraining01.uksouth.batch.azure.com",
"poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654",
"accountName": "batchfortraining01",
"linkedServiceName": {
"referenceName": "TrainingStore01",
"type": "LinkedServiceReference"
}
}
}
}

13
Code/DataFactory/linkedService/EmailSenderFunction.json
View File

@ -1,13 +0,0 @@
{
"name": "EmailSenderFunction",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureFunction",
"typeProperties": {
"functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net",
"authentication": "Anonymous",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9jZGYyMmI4Ni0zNTk2LTRkNzktYjdmOC1kNGNiMjUyNDZmYjUiDQp9"
}
}
}

16
Code/DataFactory/linkedService/GenericKeys.json
View File

@ -1,16 +0,0 @@
{
"name": "GenericKeys",
"properties": {
"type": "AzureKeyVault",
"parameters": {
"baseUrl": {
"type": "String"
}
},
"annotations": [],
"typeProperties": {
"baseUrl": "@{linkedService().baseUrl}"
},
"description": "https://TrainingKeys01.vault.azure.net"
}
}

12
Code/DataFactory/linkedService/HTTP_AWGitHub.json
View File

@ -1,12 +0,0 @@
{
"name": "HTTP_AWGitHub",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}

12
Code/DataFactory/linkedService/HTTP_AWProduct.json
View File

@ -1,12 +0,0 @@
{
"name": "HTTP_AWProduct",
"properties": {
"annotations": [],
"type": "HttpServer",
"typeProperties": {
"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
"enableServerCertificateValidation": true,
"authenticationType": "Anonymous"
}
}
}

24
Code/DataFactory/linkedService/LaptopFiles.json
View File

@ -1,24 +0,0 @@
{
"name": "LaptopFiles",
"properties": {
"annotations": [],
"type": "FileServer",
"typeProperties": {
"host": "C:\\ADFRoot\\",
"userId": "ADFIRUser",
"password": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ADFIRUser"
}
},
"connectVia": {
"referenceName": "Laptop",
"type": "IntegrationRuntimeReference"
}
},
"type": "Microsoft.DataFactory/factories/linkedservices"
}

11
Code/DataFactory/linkedService/TrainingKeys01.json
View File

@ -1,11 +0,0 @@
{
"name": "TrainingKeys01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "https://TrainingKeys01.vault.azure.net/"
}
}
}

15
Code/DataFactory/linkedService/TrainingKeys01withUMI.json
View File

@ -1,15 +0,0 @@
{
"name": "TrainingKeys01withUMI",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureKeyVault",
"typeProperties": {
"baseUrl": "https://TrainingKeys01.vault.azure.net/",
"credential": {
"referenceName": "TrainingCredential01",
"type": "CredentialReference"
}
}
}
}

12
Code/DataFactory/linkedService/TrainingStore01.json
View File

@ -1,12 +0,0 @@
{
"name": "TrainingStore01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobStorage",
"typeProperties": {
"connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9"
}
}
}

18
Code/DataFactory/linkedService/trainingdb01.json
View File

@ -1,18 +0,0 @@
{
"name": "trainingdb01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "ConnectionString-trainingdb01"
}
}
}
}

12
Code/DataFactory/linkedService/traininglak01.json
View File

@ -1,12 +0,0 @@
{
"name": "traininglak01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9"
}
}
}

19
Code/DataFactory/linkedService/traininglake01.json
View File

@ -1,19 +0,0 @@
{
"name": "traininglake01",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"accountKey": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "TrainingKeys01",
"type": "LinkedServiceReference"
},
"secretName": "traininglake01"
}
}
}
}

12
Code/DataFactory/linkedService/traininglake01noneKV.json
View File

@ -1,12 +0,0 @@
{
"name": "traininglake01noneKV",
"type": "Microsoft.DataFactory/factories/linkedservices",
"properties": {
"annotations": [],
"type": "AzureBlobFS",
"typeProperties": {
"url": "https://traininglake01.dfs.core.windows.net",
"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9"
}
}
}

4
Code/DataFactory/managedVirtualNetwork/default.json
View File

@ -1,4 +0,0 @@
{
"name": "default",
"type": "Microsoft.DataFactory/factories/managedvirtualnetworks"
}

Some files were not shown because too many files have changed in this diff Show More