.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [mrpaulandrew]  

.gitignore

@@ -0,0 +1,7 @@
+
+Code/.vs/TrainingCode/v16/.suo
+Code/MetadataDB/trainingdb01.dbmdl
+Code/MetadataDB/trainingdb01.jfm
+*.dbmdl
+*.jfm
+Code/.vs/TrainingCode/v16/.suo

Code/Checker/ADF Checker Script v0.1.ps1

@@ -0,0 +1,935 @@
+# STEP 1:
+#Provide the location of your download ARM template file:
+$ARMTemplateFilePath = "C:\ADFRoot\arm_template.json"
+
+#STEP 2:
+#Pick how you'd like the output:
+[bool]$SummaryOutput = $true 
+[bool]$VerboseOutput = $false
+
+#STEP 3:
+#Run it.
+
+#############################################################################################
+if(-not (Test-Path -Path $ARMTemplateFilePath))
+{
+    Write-Error "ARM template file not found. Please check the path provided."
+    return
+}
+
+$Hr = "-------------------------------------------------------------------------------------------------------------------"
+Write-Host ""
+Write-Host $Hr
+Write-Host "Running checks for Data Factory ARM template:"
+Write-Host ""
+$ARMTemplateFilePath
+Write-Host ""
+
+#Parse template into ADF resource parts
+$ADF = Get-Content $ARMTemplateFilePath | ConvertFrom-Json
+$LinkedServices = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/linkedServices"}
+$Datasets = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/datasets"}
+$Pipelines = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/pipelines"}
+$Activities = $Pipelines.properties.activities #regardless of pipeline
+$DataFlows = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/dataflows"}
+$Triggers = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/triggers"}
+
+#Output variables
+$CheckNumber = 0
+$CheckDetail = ""
+$Severity = ""
+$CheckCounter = 0
+$SummaryTable = @()
+$VerboseDetailTable = @()
+
+#String helper functions
+function CleanName {
+    param (
+        [parameter(Mandatory = $true)] [String] $RawValue
+    )
+    $CleanName = $RawValue.substring($RawValue.IndexOf("/")+1, $RawValue.LastIndexOf("'") - $RawValue.IndexOf("/")-1)
+    return $CleanName
+}
+
+function CleanType {
+    param (
+        [parameter(Mandatory = $true)] [String] $RawValue
+    )
+    $CleanName = $RawValue.substring($RawValue.LastIndexOf("/")+1, $RawValue.Length - $RawValue.LastIndexOf("/")-1)
+    return $CleanName
+}
+
+#############################################################################################
+#Review resource dependants
+#############################################################################################
+$ResourcesList = New-Object System.Collections.ArrayList($null)
+$DependantsList = New-Object System.Collections.ArrayList($null)
+
+#Get resources
+ForEach($Resource in $ADF.resources)
+{
+    $ResourceName = CleanName -RawValue $Resource.name
+    $ResourceType = CleanType -RawValue $Resource.type
+    $CompleteResource =   $ResourceType + "|" + $ResourceName
+    
+    if(-not ($ResourcesList -contains $CompleteResource))
+    {
+        [void]$ResourcesList.Add($CompleteResource)
+    }
+}
+
+#Get dependants
+ForEach($Resource in $ADF.resources)# | Where-Object {$_.type -ne "Microsoft.DataFactory/factories/triggers"})
+{
+    if($Resource.dependsOn.Count -eq 1)
+    {
+        $DependantName = CleanName -RawValue $Resource.dependsOn[0].ToString()
+        $CompleteDependant = $DependantName.Replace('/','|')
+
+        if(-not ($DependantsList -contains $CompleteDependant))
+        {
+            [void]$DependantsList.Add($CompleteDependant)
+        }
+    }
+    else
+    {
+        ForEach($Dependant in $Resource.dependsOn)
+        {
+            $DependantName = CleanName -RawValue $Dependant
+            $CompleteDependant = $DependantName.Replace('/','|')
+
+            if(-not ($DependantsList -contains $CompleteDependant))
+            {
+                [void]$DependantsList.Add($CompleteDependant)
+            }
+        }
+    }
+}
+
+#Get trigger dependants
+ForEach($Resource in $Triggers)
+{
+    
+    $ResourceName = CleanName -RawValue $Resource.name
+    $ResourceType = CleanType -RawValue $Resource.type
+    $CompleteResource =   $ResourceType + "|" + $ResourceName
+
+    if($Resource.dependsOn.count -ge 1)
+    {
+        if(-not ($DependantsList -contains $CompleteResource))
+        {
+            [void]$DependantsList.Add($CompleteResource)
+        }
+    }
+}
+
+#Establish simple redundancy to use later
+$RedundantResources = $ResourcesList | Where-Object {$DependantsList -notcontains $_}
+
+#############################################################################################
+#Check for pipeline without triggers
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) without any triggers attached. Directly or indirectly."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "pipelines*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Pipeline";
+            Name = $Parts[1];
+            CheckDetail = "Does not any triggers attached.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check pipeline with an impossible execution chain.
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) with an impossible AND/OR activity execution chain."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+ForEach($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $ActivityFailureDependencies = New-Object System.Collections.ArrayList($null)
+    $ActivitySuccessDependencies = New-Object System.Collections.ArrayList($null)
+
+    #get upstream failure dependants
+    ForEach($Activity in $Pipeline.properties.activities)
+    {
+        if($Activity.dependsOn.Count -gt 1)
+        {
+            ForEach($UpStreamActivity in $Activity.dependsOn)
+            {
+                if($UpStreamActivity.dependencyConditions.Contains('Failed'))
+                {  
+                    if(-not ($ActivityFailureDependencies -contains $UpStreamActivity.activity))
+                    {
+                        [void]$ActivityFailureDependencies.Add($UpStreamActivity.activity)
+                    }
+                }
+            }
+        }
+    }
+
+    #get downstream success dependants
+    ForEach($ActivityDependant in $ActivityFailureDependencies)
+    {
+        ForEach($Activity in $Pipeline.properties.activities | Where-Object {$_.name -eq $ActivityDependant})
+        {
+            if($Activity.dependsOn.Count -ge 1)
+            {
+                ForEach($DownStreamActivity in $Activity.dependsOn)
+                {
+                    if($DownStreamActivity.dependencyConditions.Contains('Succeeded'))
+                    {                  
+                        if(-not ($ActivitySuccessDependencies -contains $DownStreamActivity.activity))
+                        {
+                            [void]$ActivitySuccessDependencies.Add($DownStreamActivity.activity)
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    #compare dependants - do they exist in both lists?
+    $Problems = $ActivityFailureDependencies | Where-Object {$ActivitySuccessDependencies -contains $_}
+    if($Problems.Count -gt 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Has an impossible AND/OR activity execution chain.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check for pipeline descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $PipelineDescription = $Pipeline.properties.description
+
+    if(([string]::IsNullOrEmpty($PipelineDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for pipelines not in folders
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) not organised into folders."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $PipelineFolder = $Pipeline.properties.folder.name
+    if(([string]::IsNullOrEmpty($PipelineFolder)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Not organised into a folder.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for pipelines without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $PipelineAnnotations = $Pipeline.properties.annotations.Count
+    if($PipelineAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for data flow descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Data Flow(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($DataFlow in $DataFlows)
+{
+    $DataFlowName = (CleanName -RawValue $DataFlow.name.ToString())
+    $DataFlowDescription = $DataFlow.properties.description
+
+    if(([string]::IsNullOrEmpty($DataFlowDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Data Flow";
+                Name = $DataFlowName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check activity timeout values
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) with timeout values still set to the service default value of 7 days."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+ForEach ($Activity in $Activities)
+{
+    $timeout = $Activity.policy.timeout
+    if(-not ([string]::IsNullOrEmpty($timeout)))
+    {        
+        if($timeout -eq "7.00:00:00")
+        {
+            $CheckCounter += 1
+            if($VerboseOutput)
+            {            
+                $VerboseDetailTable += [PSCustomObject]@{
+                    Component = "Activity";
+                    Name = $Activity.Name;
+                    CheckDetail = "Timeout policy still set to the service default value of 7 days.";
+                    Severity = $Severity
+                }
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check activity descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Activity in $Activities)
+{
+    $ActivityDescription = $Activity.description
+    if(([string]::IsNullOrEmpty($ActivityDescription)))
+    {        
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {            
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Activity";
+                Name = $Activity.Name;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check foreach activity batch size unset
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) ForEach iteration without a batch count value set."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
+{    
+    [bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
+    if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
+    {
+        $isSequential = $Activity.typeProperties.isSequential
+    }
+    $BatchCount = $Activity.typeProperties.batchCount
+
+    if(!$isSequential)
+    {
+        if(([string]::IsNullOrEmpty($BatchCount)))
+        {        
+            $CheckCounter += 1
+            if($VerboseOutput)
+            {            
+                $VerboseDetailTable += [PSCustomObject]@{
+                    Component = "Activity";
+                    Name = $Activity.Name;
+                    CheckDetail = "ForEach does not have a batch count value set.";
+                    Severity = $Severity
+                }
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+
+#############################################################################################
+#Check foreach activity batch size is less than the service maximum
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) ForEach iteration with a batch count size that is less than the service maximum."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
+{   
+    [bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
+    if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
+    {
+        $isSequential = $Activity.typeProperties.isSequential
+    }
+    $BatchCount = $Activity.typeProperties.batchCount
+
+    if(!$isSequential)
+    {
+        if($BatchCount -lt 50)
+        {        
+            $CheckCounter += 1
+            if($VerboseOutput)
+            {            
+                $VerboseDetailTable += [PSCustomObject]@{
+                    Component = "Activity";
+                    Name = $Activity.Name;
+                    CheckDetail = "ForEach has a batch size that is less than the service maximum.";
+                    Severity = $Severity
+                }
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check linked service using key vault
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) not using Azure Key Vault to store credentials."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+
+$LinkedServiceList = New-Object System.Collections.ArrayList($null)
+ForEach ($LinkedService in $LinkedServices | Where-Object {$_.properties.type -ne "AzureKeyVault"})
+{
+    $typeProperties = Get-Member -InputObject $LinkedService.properties.typeProperties -MemberType NoteProperty
+
+    ForEach($typeProperty in $typeProperties) 
+    {
+        $propValue = $LinkedService.properties.typeProperties | Select-Object -ExpandProperty $typeProperty.Name
+
+        #handle linked services with multiple type properties
+        if(([string]::IsNullOrEmpty($propValue.secretName))){
+            $LinkedServiceName = (CleanName -RawValue $LinkedService.name)
+            if(-not ($LinkedServiceList -contains $LinkedServiceName))
+            {
+                [void]$LinkedServiceList.Add($LinkedServiceName) #add linked service if secretName is missing
+            }
+        }
+        if(-not([string]::IsNullOrEmpty($propValue.secretName))){
+            $LinkedServiceName = (CleanName -RawValue $LinkedService.name)
+            [void]$LinkedServiceList.Remove($LinkedServiceName) #renove linked service if secretName is then found
+        }
+    }
+}
+$CheckCounter = $LinkedServiceList.Count
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+if($VerboseOutput)
+{  
+    ForEach ($LinkedServiceOutput in $LinkedServiceList)
+    {
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Linked Service";
+            Name = $LinkedServiceOutput;
+            CheckDetail = "Not using Key Vault to store credentials.";
+            Severity = $Severity
+        }
+    }
+}
+
+#############################################################################################
+#Check for linked services not in use
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) not used by any other resource."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "linkedServices*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Linked Service";
+            Name = $Parts[1];
+            CheckDetail = "Not used by any other resource.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check linked service descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($LinkedService in $LinkedServices)
+{
+    $LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
+    $LinkedServiceDescription = $LinkedService.properties.description
+    if(([string]::IsNullOrEmpty($LinkedServiceDescription)))
+    {        
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {            
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Linked Service";
+                Name = $LinkedServiceName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for linked service without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
+    $LinkedServiceAnnotations = $Pipeline.properties.annotations.Count
+    if($LinkedServiceAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Linked Service";
+                Name = $LinkedServiceName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for datasets not in use
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) not used by any other resource."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "datasets*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Dataset";
+            Name = $Parts[1];
+            CheckDetail = "Not used by any other resource.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check for dataset without description
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Dataset in $Datasets)
+{
+    $DatasetName = (CleanName -RawValue $Dataset.name.ToString())
+    $DatasetDescription = $Dataset.properties.description
+    if(([string]::IsNullOrEmpty($DatasetDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Dataset";
+                Name = $DatasetName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check dataset not in folders
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) not organised into folders."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Dataset in $Datasets)
+{
+    $DatasetName = (CleanName -RawValue $Dataset.name.ToString())
+    $DatasetFolder = $Dataset.properties.folder.name
+    if(([string]::IsNullOrEmpty($DatasetFolder)))
+    {        
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {            
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Dataset";
+                Name = $DatasetName;
+                CheckDetail = "Not organised into a folder.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for datasets without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Dataset in $Datasets)
+{
+    $DatasetName = (CleanName -RawValue $Dataset.name.ToString())
+    $DatasetAnnotations = $Dataset.properties.annotations.Count
+    if($DatasetAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Dataset";
+                Name = $DatasetName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for triggers not in use
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Trigger(s) not used by any other resource."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "triggers*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Trigger";
+            Name = $Parts[1];
+            CheckDetail = "Not used by any other resource.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check for trigger descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Trigger(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Trigger in $Triggers)
+{
+    $TriggerName = (CleanName -RawValue $Pipeline.name.ToString())
+    $TriggerDescription = $Trigger.properties.description
+
+    if(([string]::IsNullOrEmpty($TriggerDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Trigger";
+                Name = $TriggerName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for trigger without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Trigger(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Trigger in $Triggers)
+{
+    $TriggerName = (CleanName -RawValue $Trigger.name.ToString())
+    $TriggerAnnotations = $Trigger.properties.annotations.Count
+
+    if($TriggerAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Trigger";
+                Name = $TriggerName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+
+
+#############################################################################################
+Write-Host ""
+Write-Host $Hr
+
+if($SummaryOutput)
+{    
+    Write-Host ""
+    Write-Host "Results Summary:"
+    Write-Host ""
+    Write-Host "Checks ran against template:" $CheckNumber
+    Write-Host "Checks with issues found:" ($SummaryTable | Where-Object {$_.IssueCount -ne 0}).Count.ToString()
+    Write-Host "Total issue count:" ($SummaryTable | Measure-Object -Property IssueCount -Sum).Sum
+
+    $SummaryTable | Where-Object {$_.IssueCount -ne 0} | Format-Table @{
+        Label = "Issue Count";Expression = {$_.IssueCount}; Alignment="Center"}, @{
+        Label = "Check Details";Expression = {$_.CheckDetail}}, @{
+        Label = "Severity"
+        Expression =
+        {
+            switch ($_.Severity)
+            {
+                #https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
+                'Low' {$color = "92"; break }
+                'Medium' {$color = '93'; break }
+                'High' {$color = "31"; break }
+               default {$color = "0"}
+            }
+            $e = [char]27
+           "$e[${color}m$($_.Severity)${e}[0m"
+        }
+    }
+    Write-Host $Hr
+}
+
+if($VerboseOutput)
+{
+    Write-Host ""
+    Write-Host "Results Details:"
+    
+    $VerboseDetailTable | Format-Table @{
+        Label = "Component";Expression = {$_.Component}}, @{
+        Label = "Name";Expression = {$_.Name}}, @{
+        Label = "Check Detail";Expression = {$_.CheckDetail}}, @{
+        Label = "Severity"
+        Expression =
+        {
+            switch ($_.Severity)
+            {
+                #https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
+                'Low' {$color = "92"; break }
+                'Medium' {$color = '93'; break }
+                'High' {$color = "31"; break }
+               default {$color = "0"}
+            }
+            $e = [char]27
+           "$e[${color}m$($_.Severity)${e}[0m"
+        }
+    }
+    Write-Host $Hr
+}

Code/CustomActivity/CustomActivity.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+</Project>

Code/CustomActivity/Program.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Threading;
+
+namespace CustomActivity
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Console.WriteLine("Hello World!");
+            Thread.Sleep(120000); // sleep 3mins
+            Console.WriteLine("Done");
+        }
+    }
+}

Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.deps.json

@@ -0,0 +1,23 @@
+{
+  "runtimeTarget": {
+    "name": ".NETCoreApp,Version=v3.1",
+    "signature": ""
+  },
+  "compilationOptions": {},
+  "targets": {
+    ".NETCoreApp,Version=v3.1": {
+      "CustomActivity/1.0.0": {
+        "runtime": {
+          "CustomActivity.dll": {}
+        }
+      }
+    }
+  },
+  "libraries": {
+    "CustomActivity/1.0.0": {
+      "type": "project",
+      "serviceable": false,
+      "sha512": ""
+    }
+  }
+}

Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.runtimeconfig.dev.json

@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "additionalProbingPaths": [
+      "C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
+      "C:\\Users\\paul.andrew\\.nuget\\packages",
+      "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+    ]
+  }
+}

Code/CustomActivity/bin/Debug/netcoreapp3.1/CustomActivity.runtimeconfig.json

@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "tfm": "netcoreapp3.1",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "3.1.0"
+    }
+  }
+}

Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.deps.json

@@ -0,0 +1,23 @@
+{
+  "runtimeTarget": {
+    "name": ".NETCoreApp,Version=v3.1",
+    "signature": ""
+  },
+  "compilationOptions": {},
+  "targets": {
+    ".NETCoreApp,Version=v3.1": {
+      "CustomActivity/1.0.0": {
+        "runtime": {
+          "CustomActivity.dll": {}
+        }
+      }
+    }
+  },
+  "libraries": {
+    "CustomActivity/1.0.0": {
+      "type": "project",
+      "serviceable": false,
+      "sha512": ""
+    }
+  }
+}

Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.runtimeconfig.dev.json

@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "additionalProbingPaths": [
+      "C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
+      "C:\\Users\\paul.andrew\\.nuget\\packages",
+      "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+    ]
+  }
+}

Code/CustomActivity/bin/Release/netcoreapp3.1/CustomActivity.runtimeconfig.json

@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "tfm": "netcoreapp3.1",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "3.1.0"
+    }
+  }
+}

Code/CustomActivity/obj/CustomActivity.csproj.nuget.dgspec.json

@@ -0,0 +1,66 @@
+{
+  "format": 1,
+  "restore": {
+    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {}
+  },
+  "projects": {
+    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {
+      "version": "1.0.0",
+      "restore": {
+        "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+        "projectName": "CustomActivity",
+        "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+        "packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
+        "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
+        "projectStyle": "PackageReference",
+        "fallbackFolders": [
+          "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+        ],
+        "configFilePaths": [
+          "C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+        ],
+        "originalTargetFrameworks": [
+          "netcoreapp3.1"
+        ],
+        "sources": {
+          "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+          "https://api.nuget.org/v3/index.json": {}
+        },
+        "frameworks": {
+          "netcoreapp3.1": {
+            "targetAlias": "netcoreapp3.1",
+            "projectReferences": {}
+          }
+        },
+        "warningProperties": {
+          "warnAsError": [
+            "NU1605"
+          ]
+        }
+      },
+      "frameworks": {
+        "netcoreapp3.1": {
+          "targetAlias": "netcoreapp3.1",
+          "imports": [
+            "net461",
+            "net462",
+            "net47",
+            "net471",
+            "net472",
+            "net48"
+          ],
+          "assetTargetFallback": true,
+          "warn": true,
+          "frameworkReferences": {
+            "Microsoft.NETCore.App": {
+              "privateAssets": "all"
+            }
+          },
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
+        }
+      }
+    }
+  }
+}

Code/CustomActivity/obj/CustomActivity.csproj.nuget.g.props

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
+    <RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
+    <ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
+    <NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
+    <NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\paul.andrew\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
+    <NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
+    <NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">5.11.1</NuGetToolVersion>
+  </PropertyGroup>
+  <ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <SourceRoot Include="C:\Users\paul.andrew\.nuget\packages\" />
+    <SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
+  </ItemGroup>
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup>
+</Project>

Code/CustomActivity/obj/CustomActivity.csproj.nuget.g.targets

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup>
+</Project>

Code/CustomActivity/obj/Debug/netcoreapp3.1/.NETCoreApp,Version=v3.1.AssemblyAttributes.cs

@@ -0,0 +1,4 @@
+// <autogenerated />
+using System;
+using System.Reflection;
+[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.AssemblyInfo.cs

@@ -0,0 +1,23 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by a tool.
+//     Runtime Version:4.0.30319.42000
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+using System;
+using System.Reflection;
+
+[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
+[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
+[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
+[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
+
+// Generated by the MSBuild WriteCodeFragment class.
+

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.AssemblyInfoInputs.cache

@@ -0,0 +1 @@
+0f25b4827a277d65f1c7ea5fed54274284915748

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig

@@ -0,0 +1,3 @@
+is_global = true
+build_property.RootNamespace = CustomActivity
+build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.csproj.CoreCompileInputs.cache

@@ -0,0 +1 @@
+8c2f34a721f1ab6a8a4b2c0aa34fe087f4b6de2b

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.csproj.FileListAbsolute.txt

@@ -0,0 +1,27 @@
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.AssemblyReference.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.genruntimeconfig.cache

@@ -0,0 +1 @@
+ed079c0173d4e1bcdf78b18c3f5b795d72f77c29

Code/CustomActivity/obj/Release/netcoreapp3.1/.NETCoreApp,Version=v3.1.AssemblyAttributes.cs

@@ -0,0 +1,4 @@
+// <autogenerated />
+using System;
+using System.Reflection;
+[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]

Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.AssemblyInfo.cs

@@ -0,0 +1,23 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by a tool.
+//     Runtime Version:4.0.30319.42000
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+using System;
+using System.Reflection;
+
+[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
+[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
+[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
+[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
+
+// Generated by the MSBuild WriteCodeFragment class.
+

Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.AssemblyInfoInputs.cache

@@ -0,0 +1 @@
+0f1742c03e055fa8f4a7b3c24bb888fbf0b29ac5

Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig

@@ -0,0 +1,3 @@
+is_global = true
+build_property.RootNamespace = CustomActivity
+build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\

Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.csproj.CoreCompileInputs.cache

@@ -0,0 +1 @@
+f20592bb3edc01e817172d91bf0bacb295339ec6

Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.csproj.FileListAbsolute.txt

@@ -0,0 +1,13 @@
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.exe
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.deps.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.genruntimeconfig.cache

Code/CustomActivity/obj/Release/netcoreapp3.1/CustomActivity.genruntimeconfig.cache

@@ -0,0 +1 @@
+eca3b3f0408037e0ebe5a5c84e49699310580454

Code/CustomActivity/obj/project.assets.json

@@ -0,0 +1,72 @@
+{
+  "version": 3,
+  "targets": {
+    ".NETCoreApp,Version=v3.1": {}
+  },
+  "libraries": {},
+  "projectFileDependencyGroups": {
+    ".NETCoreApp,Version=v3.1": []
+  },
+  "packageFolders": {
+    "C:\\Users\\paul.andrew\\.nuget\\packages\\": {},
+    "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages": {}
+  },
+  "project": {
+    "version": "1.0.0",
+    "restore": {
+      "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+      "projectName": "CustomActivity",
+      "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+      "packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
+      "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
+      "projectStyle": "PackageReference",
+      "fallbackFolders": [
+        "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+      ],
+      "configFilePaths": [
+        "C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
+        "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+        "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+      ],
+      "originalTargetFrameworks": [
+        "netcoreapp3.1"
+      ],
+      "sources": {
+        "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+        "https://api.nuget.org/v3/index.json": {}
+      },
+      "frameworks": {
+        "netcoreapp3.1": {
+          "targetAlias": "netcoreapp3.1",
+          "projectReferences": {}
+        }
+      },
+      "warningProperties": {
+        "warnAsError": [
+          "NU1605"
+        ]
+      }
+    },
+    "frameworks": {
+      "netcoreapp3.1": {
+        "targetAlias": "netcoreapp3.1",
+        "imports": [
+          "net461",
+          "net462",
+          "net47",
+          "net471",
+          "net472",
+          "net48"
+        ],
+        "assetTargetFallback": true,
+        "warn": true,
+        "frameworkReferences": {
+          "Microsoft.NETCore.App": {
+            "privateAssets": "all"
+          }
+        },
+        "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
+      }
+    }
+  }
+}

Code/CustomActivity/obj/project.nuget.cache

@@ -0,0 +1,8 @@
+{
+  "version": 2,
+  "dgSpecHash": "f0p4CwgurPXk2v0yD5sG6nLAvje2rr5wHxFdncruLvsOGiK3Q2I9TnSpBC64Z+fnkevGX0xQTI2XPAbWzSUW3A==",
+  "success": true,
+  "projectFilePath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+  "expectedPackageFiles": [],
+  "logs": []
+}

Code/DataFactory/credential/TrainingCredential01.json

@@ -0,0 +1,9 @@
+{
+	"name": "TrainingCredential01",
+	"properties": {
+		"type": "ManagedIdentity",
+		"typeProperties": {
+			"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
+		}
+	}
+}

Code/DataFactory/dataflow/MappingOrderAggregation.json

@@ -0,0 +1,42 @@
+{
+	"name": "MappingOrderAggregation",
+	"properties": {
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderHeaderParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderHeader"
+				},
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderDetailLinesParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderLineDetails"
+				}
+			],
+			"sinks": [
+				{
+					"dataset": {
+						"referenceName": "TableOrderSummary",
+						"type": "DatasetReference"
+					},
+					"name": "OrderSummary"
+				}
+			],
+			"transformations": [
+				{
+					"name": "JoinHeaderToLineDetails"
+				},
+				{
+					"name": "OrderLineCount"
+				}
+			],
+			"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
+		}
+	}
+}

Code/DataFactory/dataflow/MappingOrderAggregationWithParam.json

@@ -0,0 +1,45 @@
+{
+	"name": "MappingOrderAggregationWithParam",
+	"properties": {
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderHeaderParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderHeader"
+				},
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderDetailLinesParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderLineDetails"
+				}
+			],
+			"sinks": [
+				{
+					"dataset": {
+						"referenceName": "TableOrderSummary",
+						"type": "DatasetReference"
+					},
+					"name": "OrderSummary"
+				}
+			],
+			"transformations": [
+				{
+					"name": "JoinHeaderToLineDetails"
+				},
+				{
+					"name": "OrderLineCount"
+				},
+				{
+					"name": "AddAuditColum"
+				}
+			],
+			"script": "parameters{\n\tAuditColumn as string\n}\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tmatchType:'exact',\n\tignoreSpaces: false,\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount derive(AuditValue = $AuditColumn) ~> AddAuditColum\nAddAuditColum sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
+		}
+	}
+}

Code/DataFactory/dataflow/UpdateProductDimension.json

@@ -0,0 +1,165 @@
+{
+	"name": "UpdateProductDimension",
+	"properties": {
+		"folder": {
+			"name": "Labs"
+		},
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "Product"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductSubcategory"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductCategory"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "ADLS_saintegrationpipelines",
+						"type": "LinkedServiceReference"
+					},
+					"name": "WriteToDataLake"
+				}
+			],
+			"transformations": [
+				{
+					"name": "SelectProductColumns"
+				},
+				{
+					"name": "SelectSubcategoryColumns"
+				},
+				{
+					"name": "SelectCategoryColumns"
+				},
+				{
+					"name": "LookupProductCategory"
+				},
+				{
+					"name": "LookupProductSubcategory"
+				},
+				{
+					"name": "RemoveDuplicateColumns"
+				}
+			],
+			"scriptLines": [
+				"source(output(",
+				"          ProductId as integer,",
+				"          Product as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as boolean,",
+				"          {_col4_} as boolean,",
+				"          {_col5_} as string,",
+				"          {_col6_} as short,",
+				"          {_col7_} as short,",
+				"          {_col8_} as double,",
+				"          {_col9_} as double,",
+				"          {_col10_} as string,",
+				"          {_col11_} as string,",
+				"          {_col12_} as string,",
+				"          {_col13_} as double,",
+				"          {_col14_} as short,",
+				"          {_col15_} as string,",
+				"          {_col16_} as string,",
+				"          {_col17_} as string,",
+				"          SubcategoryId as integer,",
+				"          {_col19_} as short,",
+				"          {_col20_} as timestamp,",
+				"          {_col21_} as timestamp,",
+				"          {_col22_} as string,",
+				"          {_col23_} as string,",
+				"          {_col24_} as string",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> Product",
+				"source(output(",
+				"          SubcategoryId as integer,",
+				"          CategoryId as integer,",
+				"          Subcategory as string,",
+				"          {_col3_} as string,",
+				"          {_col4_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductSubcategory",
+				"source(output(",
+				"          CategoryId as integer,",
+				"          Category as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductCategory",
+				"Product select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectProductColumns",
+				"ProductSubcategory select(mapColumn(",
+				"          SubcategoryId,",
+				"          CategoryId,",
+				"          Subcategory",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
+				"ProductCategory select(mapColumn(",
+				"          CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
+				"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductCategory",
+				"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductSubcategory",
+				"LookupProductSubcategory select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId = SelectProductColumns@SubcategoryId,",
+				"          SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
+				"          CategoryId = SelectSubcategoryColumns@CategoryId,",
+				"          Subcategory,",
+				"          CategoryId = SelectCategoryColumns@CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
+				"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'parquet',",
+				"     fileSystem: 'lakeroot',",
+				"     folderPath: 'Conformed/DimProduct',",
+				"     truncate: true,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> WriteToDataLake"
+			]
+		}
+	}
+}

Code/DataFactory/dataflow/UpdateProductDimension_Sorted.json

@@ -0,0 +1,169 @@
+{
+	"name": "UpdateProductDimension_Sorted",
+	"properties": {
+		"folder": {
+			"name": "Labs"
+		},
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "Product"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductSubcategory"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductCategory"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "ADLS_saintegrationpipelines",
+						"type": "LinkedServiceReference"
+					},
+					"name": "WriteToDataLake"
+				}
+			],
+			"transformations": [
+				{
+					"name": "SelectProductColumns"
+				},
+				{
+					"name": "SelectSubcategoryColumns"
+				},
+				{
+					"name": "SelectCategoryColumns"
+				},
+				{
+					"name": "LookupProductCategory"
+				},
+				{
+					"name": "LookupProductSubcategory"
+				},
+				{
+					"name": "RemoveDuplicateColumns"
+				},
+				{
+					"name": "SortBySubcategory"
+				}
+			],
+			"scriptLines": [
+				"source(output(",
+				"          ProductId as integer,",
+				"          Product as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as boolean,",
+				"          {_col4_} as boolean,",
+				"          {_col5_} as string,",
+				"          {_col6_} as short,",
+				"          {_col7_} as short,",
+				"          {_col8_} as double,",
+				"          {_col9_} as double,",
+				"          {_col10_} as string,",
+				"          {_col11_} as string,",
+				"          {_col12_} as string,",
+				"          {_col13_} as double,",
+				"          {_col14_} as short,",
+				"          {_col15_} as string,",
+				"          {_col16_} as string,",
+				"          {_col17_} as string,",
+				"          SubcategoryId as integer,",
+				"          {_col19_} as short,",
+				"          {_col20_} as timestamp,",
+				"          {_col21_} as timestamp,",
+				"          {_col22_} as string,",
+				"          {_col23_} as string,",
+				"          {_col24_} as string",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> Product",
+				"source(output(",
+				"          SubcategoryId as integer,",
+				"          CategoryId as integer,",
+				"          Subcategory as string,",
+				"          {_col3_} as string,",
+				"          {_col4_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductSubcategory",
+				"source(output(",
+				"          CategoryId as integer,",
+				"          Category as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductCategory",
+				"Product select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectProductColumns",
+				"ProductSubcategory select(mapColumn(",
+				"          SubcategoryId,",
+				"          CategoryId,",
+				"          Subcategory",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
+				"ProductCategory select(mapColumn(",
+				"          CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
+				"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductCategory",
+				"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductSubcategory",
+				"LookupProductSubcategory select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId = SelectProductColumns@SubcategoryId,",
+				"          SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
+				"          CategoryId = SelectSubcategoryColumns@CategoryId,",
+				"          Subcategory,",
+				"          CategoryId = SelectCategoryColumns@CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
+				"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
+				"SortBySubcategory sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'parquet',",
+				"     fileSystem: 'lakeroot',",
+				"     folderPath: 'Conformed/DimProduct',",
+				"     truncate: true,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> WriteToDataLake"
+			]
+		}
+	}
+}

Code/DataFactory/dataflow/WranglingOrderAggregation.json

@@ -0,0 +1,19 @@
+{
+	"name": "WranglingOrderAggregation",
+	"properties": {
+		"type": "WranglingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"name": "LakeFileOrderDetailLinesParquet",
+					"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~>  LakeFileOrderDetailLinesParquet",
+					"dataset": {
+						"referenceName": "LakeFileOrderDetailLinesParquet",
+						"type": "DatasetReference"
+					}
+				}
+			],
+			"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n  AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n  Parquet = Parquet.Document(AdfDoc)\r\nin\r\n  Parquet;\r\nshared UserQuery = let\r\n  Source = LakeFileOrderDetailLinesParquet\r\nin\r\n  Source;\r\n"
+		}
+	}
+}

Code/DataFactory/dataset/ADLS_BIN_AWProduct.json

@@ -0,0 +1,22 @@
+{
+	"name": "ADLS_BIN_AWProduct",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "Product.csv",
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/ADLS_BIN_AdventureWorks.json

@@ -0,0 +1,30 @@
+{
+	"name": "ADLS_BIN_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@{dataset().Entity}.tsv",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/ADLS_PQT_AdventureWorks.json

@@ -0,0 +1,32 @@
+{
+	"name": "ADLS_PQT_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"EntityName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs2"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@{dataset().EntityName}.parquet",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	}
+}

Code/DataFactory/dataset/ADLS_TSV_AdventureWorks.json

@@ -0,0 +1,34 @@
+{
+	"name": "ADLS_TSV_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"FileName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().FileName",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			},
+			"columnDelimiter": "\t",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	}
+}

Code/DataFactory/dataset/AnyDatabaseTable.json

@@ -0,0 +1,43 @@
+{
+	"name": "AnyDatabaseTable",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "AnyDatabaseConnection",
+			"type": "LinkedServiceReference",
+			"parameters": {
+				"DBConnectionSecret": {
+					"value": "@dataset().LinkedServiceConnectionSecret",
+					"type": "Expression"
+				}
+			}
+		},
+		"parameters": {
+			"LinkedServiceConnectionSecret": {
+				"type": "string"
+			},
+			"SchemaName": {
+				"type": "string"
+			},
+			"TableName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": [],
+		"typeProperties": {
+			"schema": {
+				"value": "@dataset().SchemaName",
+				"type": "Expression"
+			},
+			"table": {
+				"value": "@dataset().TableName",
+				"type": "Expression"
+			}
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/AnyDatabaseTableAnyKeyVault.json

@@ -0,0 +1,50 @@
+{
+	"name": "AnyDatabaseTableAnyKeyVault",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
+			"type": "LinkedServiceReference",
+			"parameters": {
+				"KeyVaultURL": {
+					"value": "@dataset().KeyVaultURL",
+					"type": "Expression"
+				},
+				"KeyVaultSecretName": {
+					"value": "@dataset().LinkedServiceConnectionSecret",
+					"type": "Expression"
+				}
+			}
+		},
+		"parameters": {
+			"LinkedServiceConnectionSecret": {
+				"type": "string"
+			},
+			"SchemaName": {
+				"type": "string"
+			},
+			"TableName": {
+				"type": "string"
+			},
+			"KeyVaultURL": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": [],
+		"typeProperties": {
+			"schema": {
+				"value": "@dataset().SchemaName",
+				"type": "Expression"
+			},
+			"table": {
+				"value": "@dataset().TableName",
+				"type": "Expression"
+			}
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/GetSetMetadata.json

@@ -0,0 +1,16 @@
+{
+	"name": "GetSetMetadata",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "trainingdb01",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/HTTP_BIN_AWProduct.json

@@ -0,0 +1,19 @@
+{
+	"name": "HTTP_BIN_AWProduct",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWProduct",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/HTTP_BIN_AdventureWorks.json

@@ -0,0 +1,28 @@
+{
+	"name": "HTTP_BIN_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWGitHub",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation",
+				"relativeUrl": {
+					"value": "@{dataset().Entity}.csv",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}

Code/DataFactory/dataset/HTTP_TSV_AdventureWorks.json

@@ -0,0 +1,32 @@
+{
+	"name": "HTTP_TSV_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWGitHub",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"EntityName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs2"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation",
+				"relativeUrl": {
+					"value": "@{dataset().EntityName}.csv",
+					"type": "Expression"
+				}
+			},
+			"columnDelimiter": "\t",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	}
+}

Code/DataFactory/dataset/LakeFileOrderDetailLinesParquet.json

@@ -0,0 +1,68 @@
+{
+	"name": "LakeFileOrderDetailLinesParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01noneKV",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "OrderDetailLines.parquet",
+				"folderPath": "Raw",
+				"fileSystem": "datawarehouse"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": [
+			{
+				"name": "SalesOrderID",
+				"type": "INT32"
+			},
+			{
+				"name": "SalesOrderDetailID",
+				"type": "INT32"
+			},
+			{
+				"name": "OrderQty",
+				"type": "INT32"
+			},
+			{
+				"name": "ProductID",
+				"type": "INT32"
+			},
+			{
+				"name": "UnitPrice",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "UnitPriceDiscount",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "LineTotal",
+				"type": "DECIMAL",
+				"precision": 38,
+				"scale": 6
+			},
+			{
+				"name": "rowguid",
+				"type": "UTF8"
+			},
+			{
+				"name": "ModifiedDate",
+				"type": "INT96"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LakeFileOrderHeaderParquet.json

@@ -0,0 +1,122 @@
+{
+	"name": "LakeFileOrderHeaderParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01noneKV",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "OrderHeader.parquet",
+				"folderPath": "Raw",
+				"fileSystem": "datawarehouse"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": [
+			{
+				"name": "SalesOrderID",
+				"type": "INT32"
+			},
+			{
+				"name": "RevisionNumber",
+				"type": "INT32"
+			},
+			{
+				"name": "OrderDate",
+				"type": "INT96"
+			},
+			{
+				"name": "DueDate",
+				"type": "INT96"
+			},
+			{
+				"name": "ShipDate",
+				"type": "INT96"
+			},
+			{
+				"name": "Status",
+				"type": "INT32"
+			},
+			{
+				"name": "OnlineOrderFlag",
+				"type": "BOOLEAN"
+			},
+			{
+				"name": "SalesOrderNumber",
+				"type": "UTF8"
+			},
+			{
+				"name": "PurchaseOrderNumber",
+				"type": "UTF8"
+			},
+			{
+				"name": "AccountNumber",
+				"type": "UTF8"
+			},
+			{
+				"name": "CustomerID",
+				"type": "INT32"
+			},
+			{
+				"name": "ShipToAddressID",
+				"type": "INT32"
+			},
+			{
+				"name": "BillToAddressID",
+				"type": "INT32"
+			},
+			{
+				"name": "ShipMethod",
+				"type": "UTF8"
+			},
+			{
+				"name": "CreditCardApprovalCode",
+				"type": "UTF8"
+			},
+			{
+				"name": "SubTotal",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "TaxAmt",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "Freight",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "TotalDue",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "Comment",
+				"type": "UTF8"
+			},
+			{
+				"name": "rowguid",
+				"type": "UTF8"
+			},
+			{
+				"name": "ModifiedDate",
+				"type": "INT96"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LakeFileParquet.json

@@ -0,0 +1,39 @@
+{
+	"name": "LakeFileParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				},
+				"fileSystem": "datawarehouse"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LakeFilePersonCSV.json

@@ -0,0 +1,37 @@
+{
+	"name": "LakeFilePersonCSV",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "Person.csv",
+				"folderPath": "Landing",
+				"fileSystem": "datawarehouse"
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": [
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LakeFiles.json

@@ -0,0 +1,41 @@
+{
+	"name": "LakeFiles",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				},
+				"fileSystem": "datawarehouse"
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LakeFilesBinary.json

@@ -0,0 +1,36 @@
+{
+	"name": "LakeFilesBinary",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				},
+				"fileSystem": "datawarehouse"
+			}
+		}
+	}
+}

Code/DataFactory/dataset/LaptopFilePersonCSV.json

@@ -0,0 +1,36 @@
+{
+	"name": "LaptopFilePersonCSV",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": "Person.csv",
+				"folderPath": "ForUpload/People"
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": [
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LaptopFilesBinary.json

@@ -0,0 +1,35 @@
+{
+	"name": "LaptopFilesBinary",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}

Code/DataFactory/dataset/LaptopFilesParquet.json

@@ -0,0 +1,37 @@
+{
+	"name": "LaptopFilesParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	}
+}

Code/DataFactory/dataset/LaptopFolders.json

@@ -0,0 +1,33 @@
+{
+	"name": "LaptopFolders",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/LaptopsFiles.json

@@ -0,0 +1,40 @@
+{
+	"name": "LaptopsFiles",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/dataset/TableOrderSummary.json

@@ -0,0 +1,30 @@
+{
+	"name": "TableOrderSummary",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "trainingdb01",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": [
+			{
+				"name": "SalesOrderNumber",
+				"type": "varchar"
+			},
+			{
+				"name": "RecordCount",
+				"type": "int",
+				"precision": 10
+			}
+		],
+		"typeProperties": {
+			"schema": "dbo",
+			"table": "OrderSummary"
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}

Code/DataFactory/factory/TrainingFactoryDev.json

@@ -0,0 +1,30 @@
+{
+	"name": "TrainingFactoryDev",
+	"properties": {
+		"globalParameters": {
+			"Environment": {
+				"type": "string",
+				"value": "Dev"
+			},
+			"KeyVaultName": {
+				"type": "string",
+				"value": "trainingkeys01"
+			}
+		},
+		"globalConfigurations": {
+			"PipelineBillingEnabled": "true"
+		}
+	},
+	"location": "uksouth",
+	"identity": {
+		"type": "SystemAssigned,UserAssigned",
+		"principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f",
+		"tenantId": "929039b9-c19d-425e-993b-1e3fda155876",
+		"userAssignedIdentities": {
+			"/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": {
+				"clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85",
+				"principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91"
+			}
+		}
+	}
+}

Code/DataFactory/factory/adayfullofadf-adf.json

@@ -0,0 +1,4 @@
+{
+	"name": "adayfullofadf-adf",
+	"location": "uksouth"
+}

Code/DataFactory/integrationRuntime/AzureVM.json

@@ -0,0 +1,6 @@
+{
+	"name": "AzureVM",
+	"properties": {
+		"type": "SelfHosted"
+	}
+}

Code/DataFactory/integrationRuntime/ForDataFlowDemos.json

@@ -0,0 +1,16 @@
+{
+	"name": "ForDataFlowDemos",
+	"properties": {
+		"type": "Managed",
+		"typeProperties": {
+			"computeProperties": {
+				"location": "UK South",
+				"dataFlowProperties": {
+					"computeType": "General",
+					"coreCount": 8,
+					"timeToLive": 240
+				}
+			}
+		}
+	}
+}

Code/DataFactory/integrationRuntime/Laptop.json

@@ -0,0 +1,6 @@
+{
+	"name": "Laptop",
+	"properties": {
+		"type": "SelfHosted"
+	}
+}

Code/DataFactory/integrationRuntime/VNetEnabledIR.json

@@ -0,0 +1,21 @@
+{
+	"name": "VNetEnabledIR",
+	"properties": {
+		"type": "Managed",
+		"typeProperties": {
+			"computeProperties": {
+				"location": "UK South",
+				"dataFlowProperties": {
+					"computeType": "General",
+					"coreCount": 8,
+					"timeToLive": 0,
+					"cleanup": true
+				}
+			}
+		},
+		"managedVirtualNetwork": {
+			"type": "ManagedVirtualNetworkReference",
+			"referenceName": "default"
+		}
+	}
+}

Code/DataFactory/linkedService/ADLS_saintegrationpipelines.json

@@ -0,0 +1,18 @@
+{
+	"name": "ADLS_saintegrationpipelines",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"accountKey": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "traininglake01"
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/AnyDatabaseConnection.json

@@ -0,0 +1,26 @@
+{
+	"name": "AnyDatabaseConnection",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"parameters": {
+			"DBConnectionSecret": {
+				"type": "string"
+			}
+		},
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": {
+					"value": "@linkedService().DBConnectionSecret",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/AnyDatabaseConnectionFromAnyKeyVault.json

@@ -0,0 +1,34 @@
+{
+	"name": "AnyDatabaseConnectionFromAnyKeyVault",
+	"properties": {
+		"parameters": {
+			"KeyVaultURL": {
+				"type": "string"
+			},
+			"KeyVaultSecretName": {
+				"type": "string"
+			}
+		},
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "GenericKeys",
+					"type": "LinkedServiceReference",
+					"parameters": {
+						"baseUrl": {
+							"value": "@linkedService().KeyVaultURL",
+							"type": "Expression"
+						}
+					}
+				},
+				"secretName": {
+					"value": "@linkedService().KeyVaultSecretName",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/AzureDataLakeStorage1.json

@@ -0,0 +1,11 @@
+{
+	"name": "AzureDataLakeStorage1",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://adayfullofadfsa.dfs.core.windows.net"
+		}
+	}
+}

Code/DataFactory/linkedService/AzureSqlDatabase1.json

@@ -0,0 +1,12 @@
+{
+	"name": "AzureSqlDatabase1",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=adayfullofadf-sql.database.windows.net;initial catalog=AdventureWorks;user id=sql-admin",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFEQVlGVUxMT0ZBREYtQURGXzA3MGU1MDIyLWEzYjMtNDFmMS1hOTk3LWU2ODE3Y2VhNjdhYyINCn0="
+		}
+	}
+}

Code/DataFactory/linkedService/BatchForTraining01.json

@@ -0,0 +1,17 @@
+{
+	"name": "BatchForTraining01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBatch",
+		"typeProperties": {
+			"batchUri": "https://batchfortraining01.uksouth.batch.azure.com",
+			"poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654",
+			"accountName": "batchfortraining01",
+			"linkedServiceName": {
+				"referenceName": "TrainingStore01",
+				"type": "LinkedServiceReference"
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/EmailSenderFunction.json

@@ -0,0 +1,13 @@
+{
+	"name": "EmailSenderFunction",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureFunction",
+		"typeProperties": {
+			"functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net",
+			"authentication": "Anonymous",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9jZGYyMmI4Ni0zNTk2LTRkNzktYjdmOC1kNGNiMjUyNDZmYjUiDQp9"
+		}
+	}
+}

Code/DataFactory/linkedService/GenericKeys.json

@@ -0,0 +1,16 @@
+{
+	"name": "GenericKeys",
+	"properties": {
+		"type": "AzureKeyVault",
+		"parameters": {
+			"baseUrl": {
+				"type": "String"
+			}
+		},
+		"annotations": [],
+		"typeProperties": {
+			"baseUrl": "@{linkedService().baseUrl}"
+		},
+		"description": "https://TrainingKeys01.vault.azure.net"
+	}
+}

Code/DataFactory/linkedService/HTTP_AWGitHub.json

@@ -0,0 +1,12 @@
+{
+	"name": "HTTP_AWGitHub",
+	"properties": {
+		"annotations": [],
+		"type": "HttpServer",
+		"typeProperties": {
+			"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
+			"enableServerCertificateValidation": true,
+			"authenticationType": "Anonymous"
+		}
+	}
+}

Code/DataFactory/linkedService/HTTP_AWProduct.json

@@ -0,0 +1,12 @@
+{
+	"name": "HTTP_AWProduct",
+	"properties": {
+		"annotations": [],
+		"type": "HttpServer",
+		"typeProperties": {
+			"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
+			"enableServerCertificateValidation": true,
+			"authenticationType": "Anonymous"
+		}
+	}
+}

Code/DataFactory/linkedService/LaptopFiles.json

@@ -0,0 +1,24 @@
+{
+	"name": "LaptopFiles",
+	"properties": {
+		"annotations": [],
+		"type": "FileServer",
+		"typeProperties": {
+			"host": "C:\\ADFRoot\\",
+			"userId": "ADFIRUser",
+			"password": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "ADFIRUser"
+			}
+		},
+		"connectVia": {
+			"referenceName": "Laptop",
+			"type": "IntegrationRuntimeReference"
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/linkedservices"
+}

Code/DataFactory/linkedService/TrainingKeys01.json

@@ -0,0 +1,11 @@
+{
+	"name": "TrainingKeys01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureKeyVault",
+		"typeProperties": {
+			"baseUrl": "https://TrainingKeys01.vault.azure.net/"
+		}
+	}
+}

Code/DataFactory/linkedService/TrainingKeys01withUMI.json

@@ -0,0 +1,15 @@
+{
+	"name": "TrainingKeys01withUMI",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureKeyVault",
+		"typeProperties": {
+			"baseUrl": "https://TrainingKeys01.vault.azure.net/",
+			"credential": {
+				"referenceName": "TrainingCredential01",
+				"type": "CredentialReference"
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/TrainingStore01.json

@@ -0,0 +1,12 @@
+{
+	"name": "TrainingStore01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobStorage",
+		"typeProperties": {
+			"connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9"
+		}
+	}
+}

Code/DataFactory/linkedService/trainingdb01.json

@@ -0,0 +1,18 @@
+{
+	"name": "trainingdb01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "ConnectionString-trainingdb01"
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/traininglak01.json

@@ -0,0 +1,12 @@
+{
+	"name": "traininglak01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9"
+		}
+	}
+}

Code/DataFactory/linkedService/traininglake01.json

@@ -0,0 +1,19 @@
+{
+	"name": "traininglake01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"accountKey": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "traininglake01"
+			}
+		}
+	}
+}

Code/DataFactory/linkedService/traininglake01noneKV.json

@@ -0,0 +1,12 @@
+{
+	"name": "traininglake01noneKV",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9"
+		}
+	}
+}

Code/DataFactory/managedVirtualNetwork/default.json

@@ -0,0 +1,4 @@
+{
+	"name": "default",
+	"type": "Microsoft.DataFactory/factories/managedvirtualnetworks"
+}