Revised labs

Updating linkedService: LaptopFiles
2023-05-31 09:09:28 +01:00 · 2023-04-28 09:52:56 +01:00 · 2023-04-28 09:49:35 +01:00 · 2023-04-28 09:47:45 +01:00 · 2023-04-28 09:41:19 +01:00 · 2023-04-28 09:35:02 +01:00
771 changed files with 27058 additions and 9302 deletions
@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto
@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [mrpaulandrew]  
@@ -0,0 +1,935 @@
+# STEP 1:
+#Provide the location of your download ARM template file:
+$ARMTemplateFilePath = "C:\ADFRoot\arm_template.json"
+
+#STEP 2:
+#Pick how you'd like the output:
+[bool]$SummaryOutput = $true 
+[bool]$VerboseOutput = $false
+
+#STEP 3:
+#Run it.
+
+#############################################################################################
+if(-not (Test-Path -Path $ARMTemplateFilePath))
+{
+    Write-Error "ARM template file not found. Please check the path provided."
+    return
+}
+
+$Hr = "-------------------------------------------------------------------------------------------------------------------"
+Write-Host ""
+Write-Host $Hr
+Write-Host "Running checks for Data Factory ARM template:"
+Write-Host ""
+$ARMTemplateFilePath
+Write-Host ""
+
+#Parse template into ADF resource parts
+$ADF = Get-Content $ARMTemplateFilePath | ConvertFrom-Json
+$LinkedServices = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/linkedServices"}
+$Datasets = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/datasets"}
+$Pipelines = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/pipelines"}
+$Activities = $Pipelines.properties.activities #regardless of pipeline
+$DataFlows = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/dataflows"}
+$Triggers = $ADF.resources | Where-Object {$_.type -eq "Microsoft.DataFactory/factories/triggers"}
+
+#Output variables
+$CheckNumber = 0
+$CheckDetail = ""
+$Severity = ""
+$CheckCounter = 0
+$SummaryTable = @()
+$VerboseDetailTable = @()
+
+#String helper functions
+function CleanName {
+    param (
+        [parameter(Mandatory = $true)] [String] $RawValue
+    )
+    $CleanName = $RawValue.substring($RawValue.IndexOf("/")+1, $RawValue.LastIndexOf("'") - $RawValue.IndexOf("/")-1)
+    return $CleanName
+}
+
+function CleanType {
+    param (
+        [parameter(Mandatory = $true)] [String] $RawValue
+    )
+    $CleanName = $RawValue.substring($RawValue.LastIndexOf("/")+1, $RawValue.Length - $RawValue.LastIndexOf("/")-1)
+    return $CleanName
+}
+
+#############################################################################################
+#Review resource dependants
+#############################################################################################
+$ResourcesList = New-Object System.Collections.ArrayList($null)
+$DependantsList = New-Object System.Collections.ArrayList($null)
+
+#Get resources
+ForEach($Resource in $ADF.resources)
+{
+    $ResourceName = CleanName -RawValue $Resource.name
+    $ResourceType = CleanType -RawValue $Resource.type
+    $CompleteResource =   $ResourceType + "|" + $ResourceName
+    
+    if(-not ($ResourcesList -contains $CompleteResource))
+    {
+        [void]$ResourcesList.Add($CompleteResource)
+    }
+}
+
+#Get dependants
+ForEach($Resource in $ADF.resources)# | Where-Object {$_.type -ne "Microsoft.DataFactory/factories/triggers"})
+{
+    if($Resource.dependsOn.Count -eq 1)
+    {
+        $DependantName = CleanName -RawValue $Resource.dependsOn[0].ToString()
+        $CompleteDependant = $DependantName.Replace('/','|')
+
+        if(-not ($DependantsList -contains $CompleteDependant))
+        {
+            [void]$DependantsList.Add($CompleteDependant)
+        }
+    }
+    else
+    {
+        ForEach($Dependant in $Resource.dependsOn)
+        {
+            $DependantName = CleanName -RawValue $Dependant
+            $CompleteDependant = $DependantName.Replace('/','|')
+
+            if(-not ($DependantsList -contains $CompleteDependant))
+            {
+                [void]$DependantsList.Add($CompleteDependant)
+            }
+        }
+    }
+}
+
+#Get trigger dependants
+ForEach($Resource in $Triggers)
+{
+    
+    $ResourceName = CleanName -RawValue $Resource.name
+    $ResourceType = CleanType -RawValue $Resource.type
+    $CompleteResource =   $ResourceType + "|" + $ResourceName
+
+    if($Resource.dependsOn.count -ge 1)
+    {
+        if(-not ($DependantsList -contains $CompleteResource))
+        {
+            [void]$DependantsList.Add($CompleteResource)
+        }
+    }
+}
+
+#Establish simple redundancy to use later
+$RedundantResources = $ResourcesList | Where-Object {$DependantsList -notcontains $_}
+
+#############################################################################################
+#Check for pipeline without triggers
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) without any triggers attached. Directly or indirectly."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "pipelines*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Pipeline";
+            Name = $Parts[1];
+            CheckDetail = "Does not any triggers attached.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check pipeline with an impossible execution chain.
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) with an impossible AND/OR activity execution chain."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+ForEach($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $ActivityFailureDependencies = New-Object System.Collections.ArrayList($null)
+    $ActivitySuccessDependencies = New-Object System.Collections.ArrayList($null)
+
+    #get upstream failure dependants
+    ForEach($Activity in $Pipeline.properties.activities)
+    {
+        if($Activity.dependsOn.Count -gt 1)
+        {
+            ForEach($UpStreamActivity in $Activity.dependsOn)
+            {
+                if($UpStreamActivity.dependencyConditions.Contains('Failed'))
+                {  
+                    if(-not ($ActivityFailureDependencies -contains $UpStreamActivity.activity))
+                    {
+                        [void]$ActivityFailureDependencies.Add($UpStreamActivity.activity)
+                    }
+                }
+            }
+        }
+    }
+
+    #get downstream success dependants
+    ForEach($ActivityDependant in $ActivityFailureDependencies)
+    {
+        ForEach($Activity in $Pipeline.properties.activities | Where-Object {$_.name -eq $ActivityDependant})
+        {
+            if($Activity.dependsOn.Count -ge 1)
+            {
+                ForEach($DownStreamActivity in $Activity.dependsOn)
+                {
+                    if($DownStreamActivity.dependencyConditions.Contains('Succeeded'))
+                    {                  
+                        if(-not ($ActivitySuccessDependencies -contains $DownStreamActivity.activity))
+                        {
+                            [void]$ActivitySuccessDependencies.Add($DownStreamActivity.activity)
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    #compare dependants - do they exist in both lists?
+    $Problems = $ActivityFailureDependencies | Where-Object {$ActivitySuccessDependencies -contains $_}
+    if($Problems.Count -gt 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Has an impossible AND/OR activity execution chain.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check for pipeline descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $PipelineDescription = $Pipeline.properties.description
+
+    if(([string]::IsNullOrEmpty($PipelineDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for pipelines not in folders
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) not organised into folders."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $PipelineFolder = $Pipeline.properties.folder.name
+    if(([string]::IsNullOrEmpty($PipelineFolder)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Not organised into a folder.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for pipelines without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Pipeline(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $PipelineName = (CleanName -RawValue $Pipeline.name.ToString())
+    $PipelineAnnotations = $Pipeline.properties.annotations.Count
+    if($PipelineAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Pipeline";
+                Name = $PipelineName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for data flow descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Data Flow(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($DataFlow in $DataFlows)
+{
+    $DataFlowName = (CleanName -RawValue $DataFlow.name.ToString())
+    $DataFlowDescription = $DataFlow.properties.description
+
+    if(([string]::IsNullOrEmpty($DataFlowDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Data Flow";
+                Name = $DataFlowName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check activity timeout values
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) with timeout values still set to the service default value of 7 days."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+ForEach ($Activity in $Activities)
+{
+    $timeout = $Activity.policy.timeout
+    if(-not ([string]::IsNullOrEmpty($timeout)))
+    {        
+        if($timeout -eq "7.00:00:00")
+        {
+            $CheckCounter += 1
+            if($VerboseOutput)
+            {            
+                $VerboseDetailTable += [PSCustomObject]@{
+                    Component = "Activity";
+                    Name = $Activity.Name;
+                    CheckDetail = "Timeout policy still set to the service default value of 7 days.";
+                    Severity = $Severity
+                }
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check activity descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Activity in $Activities)
+{
+    $ActivityDescription = $Activity.description
+    if(([string]::IsNullOrEmpty($ActivityDescription)))
+    {        
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {            
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Activity";
+                Name = $Activity.Name;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check foreach activity batch size unset
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) ForEach iteration without a batch count value set."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
+{    
+    [bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
+    if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
+    {
+        $isSequential = $Activity.typeProperties.isSequential
+    }
+    $BatchCount = $Activity.typeProperties.batchCount
+
+    if(!$isSequential)
+    {
+        if(([string]::IsNullOrEmpty($BatchCount)))
+        {        
+            $CheckCounter += 1
+            if($VerboseOutput)
+            {            
+                $VerboseDetailTable += [PSCustomObject]@{
+                    Component = "Activity";
+                    Name = $Activity.Name;
+                    CheckDetail = "ForEach does not have a batch count value set.";
+                    Severity = $Severity
+                }
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+
+#############################################################################################
+#Check foreach activity batch size is less than the service maximum
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Activitie(s) ForEach iteration with a batch count size that is less than the service maximum."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach ($Activity in $Activities | Where-Object {$_.type -eq "ForEach"})
+{   
+    [bool]$isSequential = $false #attribute may only exist if changed, assume not present in arm template
+    if((-not [string]::IsNullOrEmpty($Activity.typeProperties.isSequential)))
+    {
+        $isSequential = $Activity.typeProperties.isSequential
+    }
+    $BatchCount = $Activity.typeProperties.batchCount
+
+    if(!$isSequential)
+    {
+        if($BatchCount -lt 50)
+        {        
+            $CheckCounter += 1
+            if($VerboseOutput)
+            {            
+                $VerboseDetailTable += [PSCustomObject]@{
+                    Component = "Activity";
+                    Name = $Activity.Name;
+                    CheckDetail = "ForEach has a batch size that is less than the service maximum.";
+                    Severity = $Severity
+                }
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check linked service using key vault
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) not using Azure Key Vault to store credentials."
+Write-Host "Running check... " $CheckDetail
+$Severity = "High"
+
+$LinkedServiceList = New-Object System.Collections.ArrayList($null)
+ForEach ($LinkedService in $LinkedServices | Where-Object {$_.properties.type -ne "AzureKeyVault"})
+{
+    $typeProperties = Get-Member -InputObject $LinkedService.properties.typeProperties -MemberType NoteProperty
+
+    ForEach($typeProperty in $typeProperties) 
+    {
+        $propValue = $LinkedService.properties.typeProperties | Select-Object -ExpandProperty $typeProperty.Name
+
+        #handle linked services with multiple type properties
+        if(([string]::IsNullOrEmpty($propValue.secretName))){
+            $LinkedServiceName = (CleanName -RawValue $LinkedService.name)
+            if(-not ($LinkedServiceList -contains $LinkedServiceName))
+            {
+                [void]$LinkedServiceList.Add($LinkedServiceName) #add linked service if secretName is missing
+            }
+        }
+        if(-not([string]::IsNullOrEmpty($propValue.secretName))){
+            $LinkedServiceName = (CleanName -RawValue $LinkedService.name)
+            [void]$LinkedServiceList.Remove($LinkedServiceName) #renove linked service if secretName is then found
+        }
+    }
+}
+$CheckCounter = $LinkedServiceList.Count
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+if($VerboseOutput)
+{  
+    ForEach ($LinkedServiceOutput in $LinkedServiceList)
+    {
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Linked Service";
+            Name = $LinkedServiceOutput;
+            CheckDetail = "Not using Key Vault to store credentials.";
+            Severity = $Severity
+        }
+    }
+}
+
+#############################################################################################
+#Check for linked services not in use
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) not used by any other resource."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "linkedServices*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Linked Service";
+            Name = $Parts[1];
+            CheckDetail = "Not used by any other resource.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check linked service descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($LinkedService in $LinkedServices)
+{
+    $LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
+    $LinkedServiceDescription = $LinkedService.properties.description
+    if(([string]::IsNullOrEmpty($LinkedServiceDescription)))
+    {        
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {            
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Linked Service";
+                Name = $LinkedServiceName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for linked service without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Linked Service(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Pipeline in $Pipelines)
+{
+    $LinkedServiceName = (CleanName -RawValue $LinkedService.name.ToString())
+    $LinkedServiceAnnotations = $Pipeline.properties.annotations.Count
+    if($LinkedServiceAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Linked Service";
+                Name = $LinkedServiceName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for datasets not in use
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) not used by any other resource."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "datasets*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Dataset";
+            Name = $Parts[1];
+            CheckDetail = "Not used by any other resource.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check for dataset without description
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Dataset in $Datasets)
+{
+    $DatasetName = (CleanName -RawValue $Dataset.name.ToString())
+    $DatasetDescription = $Dataset.properties.description
+    if(([string]::IsNullOrEmpty($DatasetDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Dataset";
+                Name = $DatasetName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check dataset not in folders
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) not organised into folders."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Dataset in $Datasets)
+{
+    $DatasetName = (CleanName -RawValue $Dataset.name.ToString())
+    $DatasetFolder = $Dataset.properties.folder.name
+    if(([string]::IsNullOrEmpty($DatasetFolder)))
+    {        
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {            
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Dataset";
+                Name = $DatasetName;
+                CheckDetail = "Not organised into a folder.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for datasets without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Dataset(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Dataset in $Datasets)
+{
+    $DatasetName = (CleanName -RawValue $Dataset.name.ToString())
+    $DatasetAnnotations = $Dataset.properties.annotations.Count
+    if($DatasetAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Dataset";
+                Name = $DatasetName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for triggers not in use
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Trigger(s) not used by any other resource."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Medium"
+ForEach($RedundantResource in $RedundantResources | Where-Object {$_ -like "triggers*"})
+{
+    $Parts = $RedundantResource.Split('|')
+
+    $CheckCounter += 1
+    if($VerboseOutput)
+    {  
+        $VerboseDetailTable += [PSCustomObject]@{
+            Component = "Trigger";
+            Name = $Parts[1];
+            CheckDetail = "Not used by any other resource.";
+            Severity = $Severity
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+    IssueCount = $CheckCounter; 
+    CheckDetail = $CheckDetail;
+    Severity = $Severity
+}
+$CheckCounter = 0
+
+#############################################################################################
+#Check for trigger descriptions
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Trigger(s) without a description value."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Trigger in $Triggers)
+{
+    $TriggerName = (CleanName -RawValue $Pipeline.name.ToString())
+    $TriggerDescription = $Trigger.properties.description
+
+    if(([string]::IsNullOrEmpty($TriggerDescription)))
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Trigger";
+                Name = $TriggerName;
+                CheckDetail = "Does not have a description.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+#############################################################################################
+#Check for trigger without annotations
+#############################################################################################
+$CheckNumber += 1
+$CheckDetail = "Trigger(s) without annotations."
+Write-Host "Running check... " $CheckDetail
+$Severity = "Low"
+ForEach ($Trigger in $Triggers)
+{
+    $TriggerName = (CleanName -RawValue $Trigger.name.ToString())
+    $TriggerAnnotations = $Trigger.properties.annotations.Count
+
+    if($TriggerAnnotations -le 0)
+    {
+        $CheckCounter += 1
+        if($VerboseOutput)
+        {  
+            $VerboseDetailTable += [PSCustomObject]@{
+                Component = "Trigger";
+                Name = $TriggerName;
+                CheckDetail = "Does not have any annotations.";
+                Severity = $Severity
+            }
+        }
+    }
+}
+$SummaryTable += [PSCustomObject]@{
+        IssueCount = $CheckCounter; 
+        CheckDetail = $CheckDetail;
+        Severity = $Severity
+    }
+$CheckCounter = 0
+
+
+
+#############################################################################################
+Write-Host ""
+Write-Host $Hr
+
+if($SummaryOutput)
+{    
+    Write-Host ""
+    Write-Host "Results Summary:"
+    Write-Host ""
+    Write-Host "Checks ran against template:" $CheckNumber
+    Write-Host "Checks with issues found:" ($SummaryTable | Where-Object {$_.IssueCount -ne 0}).Count.ToString()
+    Write-Host "Total issue count:" ($SummaryTable | Measure-Object -Property IssueCount -Sum).Sum
+
+    $SummaryTable | Where-Object {$_.IssueCount -ne 0} | Format-Table @{
+        Label = "Issue Count";Expression = {$_.IssueCount}; Alignment="Center"}, @{
+        Label = "Check Details";Expression = {$_.CheckDetail}}, @{
+        Label = "Severity"
+        Expression =
+        {
+            switch ($_.Severity)
+            {
+                #https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
+                'Low' {$color = "92"; break }
+                'Medium' {$color = '93'; break }
+                'High' {$color = "31"; break }
+               default {$color = "0"}
+            }
+            $e = [char]27
+           "$e[${color}m$($_.Severity)${e}[0m"
+        }
+    }
+    Write-Host $Hr
+}
+
+if($VerboseOutput)
+{
+    Write-Host ""
+    Write-Host "Results Details:"
+    
+    $VerboseDetailTable | Format-Table @{
+        Label = "Component";Expression = {$_.Component}}, @{
+        Label = "Name";Expression = {$_.Name}}, @{
+        Label = "Check Detail";Expression = {$_.CheckDetail}}, @{
+        Label = "Severity"
+        Expression =
+        {
+            switch ($_.Severity)
+            {
+                #https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#span-idtextformattingspanspan-idtextformattingspanspan-idtextformattingspantext-formatting
+                'Low' {$color = "92"; break }
+                'Medium' {$color = '93'; break }
+                'High' {$color = "31"; break }
+               default {$color = "0"}
+            }
+            $e = [char]27
+           "$e[${color}m$($_.Severity)${e}[0m"
+        }
+    }
+    Write-Host $Hr
+}
@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+</Project>
@@ -0,0 +1,15 @@
+using System;
+using System.Threading;
+
+namespace CustomActivity
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Console.WriteLine("Hello World!");
+            Thread.Sleep(120000); // sleep 3mins
+            Console.WriteLine("Done");
+        }
+    }
+}
@@ -0,0 +1,23 @@
+{
+  "runtimeTarget": {
+    "name": ".NETCoreApp,Version=v3.1",
+    "signature": ""
+  },
+  "compilationOptions": {},
+  "targets": {
+    ".NETCoreApp,Version=v3.1": {
+      "CustomActivity/1.0.0": {
+        "runtime": {
+          "CustomActivity.dll": {}
+        }
+      }
+    }
+  },
+  "libraries": {
+    "CustomActivity/1.0.0": {
+      "type": "project",
+      "serviceable": false,
+      "sha512": ""
+    }
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "additionalProbingPaths": [
+      "C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
+      "C:\\Users\\paul.andrew\\.nuget\\packages",
+      "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+    ]
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "tfm": "netcoreapp3.1",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "3.1.0"
+    }
+  }
+}
@@ -0,0 +1,23 @@
+{
+  "runtimeTarget": {
+    "name": ".NETCoreApp,Version=v3.1",
+    "signature": ""
+  },
+  "compilationOptions": {},
+  "targets": {
+    ".NETCoreApp,Version=v3.1": {
+      "CustomActivity/1.0.0": {
+        "runtime": {
+          "CustomActivity.dll": {}
+        }
+      }
+    }
+  },
+  "libraries": {
+    "CustomActivity/1.0.0": {
+      "type": "project",
+      "serviceable": false,
+      "sha512": ""
+    }
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "additionalProbingPaths": [
+      "C:\\Users\\paul.andrew\\.dotnet\\store\\|arch|\\|tfm|",
+      "C:\\Users\\paul.andrew\\.nuget\\packages",
+      "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+    ]
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "runtimeOptions": {
+    "tfm": "netcoreapp3.1",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "3.1.0"
+    }
+  }
+}
@@ -0,0 +1,66 @@
+{
+  "format": 1,
+  "restore": {
+    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {}
+  },
+  "projects": {
+    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {
+      "version": "1.0.0",
+      "restore": {
+        "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+        "projectName": "CustomActivity",
+        "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+        "packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
+        "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
+        "projectStyle": "PackageReference",
+        "fallbackFolders": [
+          "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+        ],
+        "configFilePaths": [
+          "C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+          "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+        ],
+        "originalTargetFrameworks": [
+          "netcoreapp3.1"
+        ],
+        "sources": {
+          "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+          "https://api.nuget.org/v3/index.json": {}
+        },
+        "frameworks": {
+          "netcoreapp3.1": {
+            "targetAlias": "netcoreapp3.1",
+            "projectReferences": {}
+          }
+        },
+        "warningProperties": {
+          "warnAsError": [
+            "NU1605"
+          ]
+        }
+      },
+      "frameworks": {
+        "netcoreapp3.1": {
+          "targetAlias": "netcoreapp3.1",
+          "imports": [
+            "net461",
+            "net462",
+            "net47",
+            "net471",
+            "net472",
+            "net48"
+          ],
+          "assetTargetFallback": true,
+          "warn": true,
+          "frameworkReferences": {
+            "Microsoft.NETCore.App": {
+              "privateAssets": "all"
+            }
+          },
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
+        }
+      }
+    }
+  }
+}
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
+    <RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
+    <ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
+    <NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
+    <NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\paul.andrew\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
+    <NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
+    <NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">5.11.1</NuGetToolVersion>
+  </PropertyGroup>
+  <ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
+    <SourceRoot Include="C:\Users\paul.andrew\.nuget\packages\" />
+    <SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
+  </ItemGroup>
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup>
+</Project>
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup>
+</Project>
@@ -0,0 +1,4 @@
+// <autogenerated />
+using System;
+using System.Reflection;
+[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]
@@ -0,0 +1,23 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by a tool.
+//     Runtime Version:4.0.30319.42000
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+using System;
+using System.Reflection;
+
+[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
+[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
+[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
+[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
+
+// Generated by the MSBuild WriteCodeFragment class.
+
@@ -0,0 +1 @@
+0f25b4827a277d65f1c7ea5fed54274284915748
@@ -0,0 +1,3 @@
+is_global = true
+build_property.RootNamespace = CustomActivity
+build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\
@@ -0,0 +1 @@
+8c2f34a721f1ab6a8a4b2c0aa34fe087f4b6de2b
@@ -0,0 +1,27 @@
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.AssemblyReference.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.exe
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.deps.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\bin\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\obj\Debug\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
@@ -0,0 +1 @@
+ed079c0173d4e1bcdf78b18c3f5b795d72f77c29
@@ -0,0 +1,4 @@
+// <autogenerated />
+using System;
+using System.Reflection;
+[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v3.1", FrameworkDisplayName = "")]
@@ -0,0 +1,23 @@
+//------------------------------------------------------------------------------
+// <auto-generated>
+//     This code was generated by a tool.
+//     Runtime Version:4.0.30319.42000
+//
+//     Changes to this file may cause incorrect behavior and will be lost if
+//     the code is regenerated.
+// </auto-generated>
+//------------------------------------------------------------------------------
+
+using System;
+using System.Reflection;
+
+[assembly: System.Reflection.AssemblyCompanyAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
+[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
+[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
+[assembly: System.Reflection.AssemblyProductAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyTitleAttribute("CustomActivity")]
+[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
+
+// Generated by the MSBuild WriteCodeFragment class.
+
@@ -0,0 +1 @@
+0f1742c03e055fa8f4a7b3c24bb888fbf0b29ac5
@@ -0,0 +1,3 @@
+is_global = true
+build_property.RootNamespace = CustomActivity
+build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\
@@ -0,0 +1 @@
+f20592bb3edc01e817172d91bf0bacb295339ec6
@@ -0,0 +1,13 @@
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.exe
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.deps.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.runtimeconfig.dev.json
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\bin\Release\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.GeneratedMSBuildEditorConfig.editorconfig
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfoInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.AssemblyInfo.cs
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.csproj.CoreCompileInputs.cache
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.dll
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.pdb
+C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\CustomActivity\obj\Release\netcoreapp3.1\CustomActivity.genruntimeconfig.cache
@@ -0,0 +1 @@
+eca3b3f0408037e0ebe5a5c84e49699310580454
@@ -0,0 +1,72 @@
+{
+  "version": 3,
+  "targets": {
+    ".NETCoreApp,Version=v3.1": {}
+  },
+  "libraries": {},
+  "projectFileDependencyGroups": {
+    ".NETCoreApp,Version=v3.1": []
+  },
+  "packageFolders": {
+    "C:\\Users\\paul.andrew\\.nuget\\packages\\": {},
+    "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages": {}
+  },
+  "project": {
+    "version": "1.0.0",
+    "restore": {
+      "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+      "projectName": "CustomActivity",
+      "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+      "packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
+      "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
+      "projectStyle": "PackageReference",
+      "fallbackFolders": [
+        "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
+      ],
+      "configFilePaths": [
+        "C:\\Users\\paul.andrew\\AppData\\Roaming\\NuGet\\NuGet.Config",
+        "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
+        "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
+      ],
+      "originalTargetFrameworks": [
+        "netcoreapp3.1"
+      ],
+      "sources": {
+        "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
+        "https://api.nuget.org/v3/index.json": {}
+      },
+      "frameworks": {
+        "netcoreapp3.1": {
+          "targetAlias": "netcoreapp3.1",
+          "projectReferences": {}
+        }
+      },
+      "warningProperties": {
+        "warnAsError": [
+          "NU1605"
+        ]
+      }
+    },
+    "frameworks": {
+      "netcoreapp3.1": {
+        "targetAlias": "netcoreapp3.1",
+        "imports": [
+          "net461",
+          "net462",
+          "net47",
+          "net471",
+          "net472",
+          "net48"
+        ],
+        "assetTargetFallback": true,
+        "warn": true,
+        "frameworkReferences": {
+          "Microsoft.NETCore.App": {
+            "privateAssets": "all"
+          }
+        },
+        "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
+      }
+    }
+  }
+}
@@ -0,0 +1,8 @@
+{
+  "version": 2,
+  "dgSpecHash": "f0p4CwgurPXk2v0yD5sG6nLAvje2rr5wHxFdncruLvsOGiK3Q2I9TnSpBC64Z+fnkevGX0xQTI2XPAbWzSUW3A==",
+  "success": true,
+  "projectFilePath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
+  "expectedPackageFiles": [],
+  "logs": []
+}
@@ -0,0 +1,9 @@
+{
+	"name": "TrainingCredential01",
+	"properties": {
+		"type": "ManagedIdentity",
+		"typeProperties": {
+			"resourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI"
+		}
+	}
+}
@@ -0,0 +1,42 @@
+{
+	"name": "MappingOrderAggregation",
+	"properties": {
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderHeaderParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderHeader"
+				},
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderDetailLinesParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderLineDetails"
+				}
+			],
+			"sinks": [
+				{
+					"dataset": {
+						"referenceName": "TableOrderSummary",
+						"type": "DatasetReference"
+					},
+					"name": "OrderSummary"
+				}
+			],
+			"transformations": [
+				{
+					"name": "JoinHeaderToLineDetails"
+				},
+				{
+					"name": "OrderLineCount"
+				}
+			],
+			"script": "source(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
+		}
+	}
+}
@@ -0,0 +1,45 @@
+{
+	"name": "MappingOrderAggregationWithParam",
+	"properties": {
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderHeaderParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderHeader"
+				},
+				{
+					"dataset": {
+						"referenceName": "LakeFileOrderDetailLinesParquet",
+						"type": "DatasetReference"
+					},
+					"name": "OrderLineDetails"
+				}
+			],
+			"sinks": [
+				{
+					"dataset": {
+						"referenceName": "TableOrderSummary",
+						"type": "DatasetReference"
+					},
+					"name": "OrderSummary"
+				}
+			],
+			"transformations": [
+				{
+					"name": "JoinHeaderToLineDetails"
+				},
+				{
+					"name": "OrderLineCount"
+				},
+				{
+					"name": "AddAuditColum"
+				}
+			],
+			"script": "parameters{\n\tAuditColumn as string\n}\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tStatus as integer,\n\t\tOnlineOrderFlag as boolean,\n\t\tSalesOrderNumber as string,\n\t\tPurchaseOrderNumber as string,\n\t\tAccountNumber as string,\n\t\tCustomerID as integer,\n\t\tShipToAddressID as integer,\n\t\tBillToAddressID as integer,\n\t\tShipMethod as string,\n\t\tCreditCardApprovalCode as string,\n\t\tSubTotal as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tTotalDue as decimal(19,4),\n\t\tComment as string,\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderHeader\nsource(output(\n\t\tSalesOrderID as integer,\n\t\tSalesOrderDetailID as integer,\n\t\tOrderQty as integer,\n\t\tProductID as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tUnitPriceDiscount as decimal(19,4),\n\t\tLineTotal as decimal(38,6),\n\t\trowguid as string,\n\t\tModifiedDate as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet',\n\tpartitionBy('hash', 1)) ~> OrderLineDetails\nOrderHeader, OrderLineDetails join(OrderHeader@SalesOrderID == OrderLineDetails@SalesOrderID,\n\tjoinType:'inner',\n\tmatchType:'exact',\n\tignoreSpaces: false,\n\tpartitionBy('hash', 1),\n\tbroadcast: 'both')~> JoinHeaderToLineDetails\nJoinHeaderToLineDetails aggregate(groupBy(SalesOrderNumber),\n\tRecordCount = count(SalesOrderDetailID),\n\tpartitionBy('roundRobin', 4)) ~> OrderLineCount\nOrderLineCount derive(AuditValue = $AuditColumn) ~> AddAuditColum\nAddAuditColum sink(allowSchemaDrift: false,\n\tvalidateSchema: false,\n\tinput(\n\t\tSalesOrderNumber as string,\n\t\tRecordCount as integer\n\t),\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\ttruncate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\terrorHandlingOption: 'stopOnFirstError',\n\tmapColumn(\n\t\tSalesOrderNumber,\n\t\tRecordCount\n\t),\n\tpartitionBy('roundRobin', 4)) ~> OrderSummary"
+		}
+	}
+}
@@ -0,0 +1,165 @@
+{
+	"name": "UpdateProductDimension",
+	"properties": {
+		"folder": {
+			"name": "Labs"
+		},
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "Product"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductSubcategory"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductCategory"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "ADLS_saintegrationpipelines",
+						"type": "LinkedServiceReference"
+					},
+					"name": "WriteToDataLake"
+				}
+			],
+			"transformations": [
+				{
+					"name": "SelectProductColumns"
+				},
+				{
+					"name": "SelectSubcategoryColumns"
+				},
+				{
+					"name": "SelectCategoryColumns"
+				},
+				{
+					"name": "LookupProductCategory"
+				},
+				{
+					"name": "LookupProductSubcategory"
+				},
+				{
+					"name": "RemoveDuplicateColumns"
+				}
+			],
+			"scriptLines": [
+				"source(output(",
+				"          ProductId as integer,",
+				"          Product as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as boolean,",
+				"          {_col4_} as boolean,",
+				"          {_col5_} as string,",
+				"          {_col6_} as short,",
+				"          {_col7_} as short,",
+				"          {_col8_} as double,",
+				"          {_col9_} as double,",
+				"          {_col10_} as string,",
+				"          {_col11_} as string,",
+				"          {_col12_} as string,",
+				"          {_col13_} as double,",
+				"          {_col14_} as short,",
+				"          {_col15_} as string,",
+				"          {_col16_} as string,",
+				"          {_col17_} as string,",
+				"          SubcategoryId as integer,",
+				"          {_col19_} as short,",
+				"          {_col20_} as timestamp,",
+				"          {_col21_} as timestamp,",
+				"          {_col22_} as string,",
+				"          {_col23_} as string,",
+				"          {_col24_} as string",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> Product",
+				"source(output(",
+				"          SubcategoryId as integer,",
+				"          CategoryId as integer,",
+				"          Subcategory as string,",
+				"          {_col3_} as string,",
+				"          {_col4_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductSubcategory",
+				"source(output(",
+				"          CategoryId as integer,",
+				"          Category as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductCategory",
+				"Product select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectProductColumns",
+				"ProductSubcategory select(mapColumn(",
+				"          SubcategoryId,",
+				"          CategoryId,",
+				"          Subcategory",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
+				"ProductCategory select(mapColumn(",
+				"          CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
+				"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductCategory",
+				"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductSubcategory",
+				"LookupProductSubcategory select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId = SelectProductColumns@SubcategoryId,",
+				"          SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
+				"          CategoryId = SelectSubcategoryColumns@CategoryId,",
+				"          Subcategory,",
+				"          CategoryId = SelectCategoryColumns@CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
+				"RemoveDuplicateColumns sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'parquet',",
+				"     fileSystem: 'lakeroot',",
+				"     folderPath: 'Conformed/DimProduct',",
+				"     truncate: true,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> WriteToDataLake"
+			]
+		}
+	}
+}
@@ -0,0 +1,169 @@
+{
+	"name": "UpdateProductDimension_Sorted",
+	"properties": {
+		"folder": {
+			"name": "Labs"
+		},
+		"type": "MappingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "Product"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductSubcategory"
+				},
+				{
+					"dataset": {
+						"referenceName": "ADLS_TSV_AdventureWorks",
+						"type": "DatasetReference"
+					},
+					"name": "ProductCategory"
+				}
+			],
+			"sinks": [
+				{
+					"linkedService": {
+						"referenceName": "ADLS_saintegrationpipelines",
+						"type": "LinkedServiceReference"
+					},
+					"name": "WriteToDataLake"
+				}
+			],
+			"transformations": [
+				{
+					"name": "SelectProductColumns"
+				},
+				{
+					"name": "SelectSubcategoryColumns"
+				},
+				{
+					"name": "SelectCategoryColumns"
+				},
+				{
+					"name": "LookupProductCategory"
+				},
+				{
+					"name": "LookupProductSubcategory"
+				},
+				{
+					"name": "RemoveDuplicateColumns"
+				},
+				{
+					"name": "SortBySubcategory"
+				}
+			],
+			"scriptLines": [
+				"source(output(",
+				"          ProductId as integer,",
+				"          Product as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as boolean,",
+				"          {_col4_} as boolean,",
+				"          {_col5_} as string,",
+				"          {_col6_} as short,",
+				"          {_col7_} as short,",
+				"          {_col8_} as double,",
+				"          {_col9_} as double,",
+				"          {_col10_} as string,",
+				"          {_col11_} as string,",
+				"          {_col12_} as string,",
+				"          {_col13_} as double,",
+				"          {_col14_} as short,",
+				"          {_col15_} as string,",
+				"          {_col16_} as string,",
+				"          {_col17_} as string,",
+				"          SubcategoryId as integer,",
+				"          {_col19_} as short,",
+				"          {_col20_} as timestamp,",
+				"          {_col21_} as timestamp,",
+				"          {_col22_} as string,",
+				"          {_col23_} as string,",
+				"          {_col24_} as string",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> Product",
+				"source(output(",
+				"          SubcategoryId as integer,",
+				"          CategoryId as integer,",
+				"          Subcategory as string,",
+				"          {_col3_} as string,",
+				"          {_col4_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductSubcategory",
+				"source(output(",
+				"          CategoryId as integer,",
+				"          Category as string,",
+				"          {_col2_} as string,",
+				"          {_col3_} as timestamp",
+				"     ),",
+				"     allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     ignoreNoFilesFound: false) ~> ProductCategory",
+				"Product select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectProductColumns",
+				"ProductSubcategory select(mapColumn(",
+				"          SubcategoryId,",
+				"          CategoryId,",
+				"          Subcategory",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectSubcategoryColumns",
+				"ProductCategory select(mapColumn(",
+				"          CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> SelectCategoryColumns",
+				"SelectSubcategoryColumns, SelectCategoryColumns lookup(SelectSubcategoryColumns@CategoryId == SelectCategoryColumns@CategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductCategory",
+				"SelectProductColumns, LookupProductCategory lookup(SelectProductColumns@SubcategoryId == SelectSubcategoryColumns@SubcategoryId,",
+				"     multiple: false,",
+				"     pickup: 'any',",
+				"     broadcast: 'auto')~> LookupProductSubcategory",
+				"LookupProductSubcategory select(mapColumn(",
+				"          ProductId,",
+				"          Product,",
+				"          SubcategoryId = SelectProductColumns@SubcategoryId,",
+				"          SubcategoryId = SelectSubcategoryColumns@SubcategoryId,",
+				"          CategoryId = SelectSubcategoryColumns@CategoryId,",
+				"          Subcategory,",
+				"          CategoryId = SelectCategoryColumns@CategoryId,",
+				"          Category",
+				"     ),",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> RemoveDuplicateColumns",
+				"RemoveDuplicateColumns sort(asc(Subcategory, false)) ~> SortBySubcategory",
+				"SortBySubcategory sink(allowSchemaDrift: true,",
+				"     validateSchema: false,",
+				"     format: 'parquet',",
+				"     fileSystem: 'lakeroot',",
+				"     folderPath: 'Conformed/DimProduct',",
+				"     truncate: true,",
+				"     umask: 0022,",
+				"     preCommands: [],",
+				"     postCommands: [],",
+				"     skipDuplicateMapInputs: true,",
+				"     skipDuplicateMapOutputs: true) ~> WriteToDataLake"
+			]
+		}
+	}
+}
@@ -0,0 +1,19 @@
+{
+	"name": "WranglingOrderAggregation",
+	"properties": {
+		"type": "WranglingDataFlow",
+		"typeProperties": {
+			"sources": [
+				{
+					"name": "LakeFileOrderDetailLinesParquet",
+					"script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tformat: 'parquet') ~>  LakeFileOrderDetailLinesParquet",
+					"dataset": {
+						"referenceName": "LakeFileOrderDetailLinesParquet",
+						"type": "DatasetReference"
+					}
+				}
+			],
+			"script": "section Section1;\r\nshared LakeFileOrderDetailLinesParquet = let\r\n  AdfDoc = Web.Contents(\"https://traininglake01.dfs.core.windows.net/datawarehouse/Raw/OrderDetailLines.parquet?sv=2018-03-28&sig=5R%2BzQI0dTqfGUYi8vVuzKhHq6DBYMX%2FYNyfH4c1BalM%3D&spr=https&se=2020-09-02T12%3A16%3A29Z&srt=sco&ss=bf&sp=rwl\"),\r\n  Parquet = Parquet.Document(AdfDoc)\r\nin\r\n  Parquet;\r\nshared UserQuery = let\r\n  Source = LakeFileOrderDetailLinesParquet\r\nin\r\n  Source;\r\n"
+		}
+	}
+}
@@ -0,0 +1,22 @@
+{
+	"name": "ADLS_BIN_AWProduct",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "Product.csv",
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			}
+		}
+	}
+}
@@ -0,0 +1,30 @@
+{
+	"name": "ADLS_BIN_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@{dataset().Entity}.tsv",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			}
+		}
+	}
+}
@@ -0,0 +1,32 @@
+{
+	"name": "ADLS_PQT_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"EntityName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs2"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@{dataset().EntityName}.parquet",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	}
+}
@@ -0,0 +1,34 @@
+{
+	"name": "ADLS_TSV_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "ADLS_saintegrationpipelines",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"FileName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().FileName",
+					"type": "Expression"
+				},
+				"folderPath": "Raw",
+				"fileSystem": "lakeroot"
+			},
+			"columnDelimiter": "\t",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	}
+}
@@ -0,0 +1,43 @@
+{
+	"name": "AnyDatabaseTable",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "AnyDatabaseConnection",
+			"type": "LinkedServiceReference",
+			"parameters": {
+				"DBConnectionSecret": {
+					"value": "@dataset().LinkedServiceConnectionSecret",
+					"type": "Expression"
+				}
+			}
+		},
+		"parameters": {
+			"LinkedServiceConnectionSecret": {
+				"type": "string"
+			},
+			"SchemaName": {
+				"type": "string"
+			},
+			"TableName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": [],
+		"typeProperties": {
+			"schema": {
+				"value": "@dataset().SchemaName",
+				"type": "Expression"
+			},
+			"table": {
+				"value": "@dataset().TableName",
+				"type": "Expression"
+			}
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,50 @@
+{
+	"name": "AnyDatabaseTableAnyKeyVault",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "AnyDatabaseConnectionFromAnyKeyVault",
+			"type": "LinkedServiceReference",
+			"parameters": {
+				"KeyVaultURL": {
+					"value": "@dataset().KeyVaultURL",
+					"type": "Expression"
+				},
+				"KeyVaultSecretName": {
+					"value": "@dataset().LinkedServiceConnectionSecret",
+					"type": "Expression"
+				}
+			}
+		},
+		"parameters": {
+			"LinkedServiceConnectionSecret": {
+				"type": "string"
+			},
+			"SchemaName": {
+				"type": "string"
+			},
+			"TableName": {
+				"type": "string"
+			},
+			"KeyVaultURL": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": [],
+		"typeProperties": {
+			"schema": {
+				"value": "@dataset().SchemaName",
+				"type": "Expression"
+			},
+			"table": {
+				"value": "@dataset().TableName",
+				"type": "Expression"
+			}
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,16 @@
+{
+	"name": "GetSetMetadata",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "trainingdb01",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,19 @@
+{
+	"name": "HTTP_BIN_AWProduct",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWProduct",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation"
+			}
+		}
+	}
+}
@@ -0,0 +1,28 @@
+{
+	"name": "HTTP_BIN_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWGitHub",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Entity": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs1"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation",
+				"relativeUrl": {
+					"value": "@{dataset().Entity}.csv",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,32 @@
+{
+	"name": "HTTP_TSV_AdventureWorks",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "HTTP_AWGitHub",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"EntityName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Labs2"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "HttpServerLocation",
+				"relativeUrl": {
+					"value": "@{dataset().EntityName}.csv",
+					"type": "Expression"
+				}
+			},
+			"columnDelimiter": "\t",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	}
+}
@@ -0,0 +1,68 @@
+{
+	"name": "LakeFileOrderDetailLinesParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01noneKV",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "OrderDetailLines.parquet",
+				"folderPath": "Raw",
+				"fileSystem": "datawarehouse"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": [
+			{
+				"name": "SalesOrderID",
+				"type": "INT32"
+			},
+			{
+				"name": "SalesOrderDetailID",
+				"type": "INT32"
+			},
+			{
+				"name": "OrderQty",
+				"type": "INT32"
+			},
+			{
+				"name": "ProductID",
+				"type": "INT32"
+			},
+			{
+				"name": "UnitPrice",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "UnitPriceDiscount",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "LineTotal",
+				"type": "DECIMAL",
+				"precision": 38,
+				"scale": 6
+			},
+			{
+				"name": "rowguid",
+				"type": "UTF8"
+			},
+			{
+				"name": "ModifiedDate",
+				"type": "INT96"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,122 @@
+{
+	"name": "LakeFileOrderHeaderParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01noneKV",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "OrderHeader.parquet",
+				"folderPath": "Raw",
+				"fileSystem": "datawarehouse"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": [
+			{
+				"name": "SalesOrderID",
+				"type": "INT32"
+			},
+			{
+				"name": "RevisionNumber",
+				"type": "INT32"
+			},
+			{
+				"name": "OrderDate",
+				"type": "INT96"
+			},
+			{
+				"name": "DueDate",
+				"type": "INT96"
+			},
+			{
+				"name": "ShipDate",
+				"type": "INT96"
+			},
+			{
+				"name": "Status",
+				"type": "INT32"
+			},
+			{
+				"name": "OnlineOrderFlag",
+				"type": "BOOLEAN"
+			},
+			{
+				"name": "SalesOrderNumber",
+				"type": "UTF8"
+			},
+			{
+				"name": "PurchaseOrderNumber",
+				"type": "UTF8"
+			},
+			{
+				"name": "AccountNumber",
+				"type": "UTF8"
+			},
+			{
+				"name": "CustomerID",
+				"type": "INT32"
+			},
+			{
+				"name": "ShipToAddressID",
+				"type": "INT32"
+			},
+			{
+				"name": "BillToAddressID",
+				"type": "INT32"
+			},
+			{
+				"name": "ShipMethod",
+				"type": "UTF8"
+			},
+			{
+				"name": "CreditCardApprovalCode",
+				"type": "UTF8"
+			},
+			{
+				"name": "SubTotal",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "TaxAmt",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "Freight",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "TotalDue",
+				"type": "DECIMAL",
+				"precision": 19,
+				"scale": 4
+			},
+			{
+				"name": "Comment",
+				"type": "UTF8"
+			},
+			{
+				"name": "rowguid",
+				"type": "UTF8"
+			},
+			{
+				"name": "ModifiedDate",
+				"type": "INT96"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,39 @@
+{
+	"name": "LakeFileParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				},
+				"fileSystem": "datawarehouse"
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,37 @@
+{
+	"name": "LakeFilePersonCSV",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": "Person.csv",
+				"folderPath": "Landing",
+				"fileSystem": "datawarehouse"
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": [
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,41 @@
+{
+	"name": "LakeFiles",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				},
+				"fileSystem": "datawarehouse"
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,36 @@
+{
+	"name": "LakeFilesBinary",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "traininglake01",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Lake"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "AzureBlobFSLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				},
+				"fileSystem": "datawarehouse"
+			}
+		}
+	}
+}
@@ -0,0 +1,36 @@
+{
+	"name": "LaptopFilePersonCSV",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": "Person.csv",
+				"folderPath": "ForUpload/People"
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": [
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			},
+			{
+				"type": "String"
+			}
+		]
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,35 @@
+{
+	"name": "LaptopFilesBinary",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "Binary",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,37 @@
+{
+	"name": "LaptopFilesParquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	}
+}
@@ -0,0 +1,33 @@
+{
+	"name": "LaptopFolders",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,40 @@
+{
+	"name": "LaptopsFiles",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "LaptopFiles",
+			"type": "LinkedServiceReference"
+		},
+		"parameters": {
+			"Directory": {
+				"type": "string"
+			},
+			"File": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Laptop"
+		},
+		"annotations": [],
+		"type": "DelimitedText",
+		"typeProperties": {
+			"location": {
+				"type": "FileServerLocation",
+				"fileName": {
+					"value": "@dataset().File",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().Directory",
+					"type": "Expression"
+				}
+			},
+			"columnDelimiter": ",",
+			"escapeChar": "\\",
+			"quoteChar": "\""
+		},
+		"schema": []
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,30 @@
+{
+	"name": "TableOrderSummary",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "trainingdb01",
+			"type": "LinkedServiceReference"
+		},
+		"folder": {
+			"name": "SQLDB"
+		},
+		"annotations": [],
+		"type": "AzureSqlTable",
+		"schema": [
+			{
+				"name": "SalesOrderNumber",
+				"type": "varchar"
+			},
+			{
+				"name": "RecordCount",
+				"type": "int",
+				"precision": 10
+			}
+		],
+		"typeProperties": {
+			"schema": "dbo",
+			"table": "OrderSummary"
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/datasets"
+}
@@ -0,0 +1,30 @@
+{
+	"name": "TrainingFactoryDev",
+	"properties": {
+		"globalParameters": {
+			"Environment": {
+				"type": "string",
+				"value": "Dev"
+			},
+			"KeyVaultName": {
+				"type": "string",
+				"value": "trainingkeys01"
+			}
+		},
+		"globalConfigurations": {
+			"PipelineBillingEnabled": "true"
+		}
+	},
+	"location": "uksouth",
+	"identity": {
+		"type": "SystemAssigned,UserAssigned",
+		"principalId": "f818bfb8-5602-4e49-9bd5-81f3855cc81f",
+		"tenantId": "929039b9-c19d-425e-993b-1e3fda155876",
+		"userAssignedIdentities": {
+			"/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourcegroups/Training/providers/Microsoft.ManagedIdentity/userAssignedIdentities/TrainingUMI": {
+				"clientId": "3f2854d7-9e2f-45db-9543-5f42db139e85",
+				"principalId": "05ac2a67-6e10-4eb2-b2fc-28e5d7c0bb91"
+			}
+		}
+	}
+}
@@ -0,0 +1,4 @@
+{
+	"name": "adayfullofadf-adf",
+	"location": "uksouth"
+}
@@ -0,0 +1,16 @@
+{
+	"name": "ForDataFlowDemos",
+	"properties": {
+		"type": "Managed",
+		"typeProperties": {
+			"computeProperties": {
+				"location": "UK South",
+				"dataFlowProperties": {
+					"computeType": "General",
+					"coreCount": 8,
+					"timeToLive": 240
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,6 @@
+{
+	"name": "Laptop",
+	"properties": {
+		"type": "SelfHosted"
+	}
+}
@@ -0,0 +1,21 @@
+{
+	"name": "VNetEnabledIR",
+	"properties": {
+		"type": "Managed",
+		"typeProperties": {
+			"computeProperties": {
+				"location": "UK South",
+				"dataFlowProperties": {
+					"computeType": "General",
+					"coreCount": 8,
+					"timeToLive": 0,
+					"cleanup": true
+				}
+			}
+		},
+		"managedVirtualNetwork": {
+			"type": "ManagedVirtualNetworkReference",
+			"referenceName": "default"
+		}
+	}
+}
@@ -0,0 +1,18 @@
+{
+	"name": "ADLS_saintegrationpipelines",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"accountKey": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "traininglake01"
+			}
+		}
+	}
+}
@@ -0,0 +1,26 @@
+{
+	"name": "AnyDatabaseConnection",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"parameters": {
+			"DBConnectionSecret": {
+				"type": "string"
+			}
+		},
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": {
+					"value": "@linkedService().DBConnectionSecret",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,34 @@
+{
+	"name": "AnyDatabaseConnectionFromAnyKeyVault",
+	"properties": {
+		"parameters": {
+			"KeyVaultURL": {
+				"type": "string"
+			},
+			"KeyVaultSecretName": {
+				"type": "string"
+			}
+		},
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "GenericKeys",
+					"type": "LinkedServiceReference",
+					"parameters": {
+						"baseUrl": {
+							"value": "@linkedService().KeyVaultURL",
+							"type": "Expression"
+						}
+					}
+				},
+				"secretName": {
+					"value": "@linkedService().KeyVaultSecretName",
+					"type": "Expression"
+				}
+			}
+		}
+	}
+}
@@ -0,0 +1,11 @@
+{
+	"name": "AzureDataLakeStorage1",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://adayfullofadfsa.dfs.core.windows.net"
+		}
+	}
+}
@@ -0,0 +1,12 @@
+{
+	"name": "AzureSqlDatabase1",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=adayfullofadf-sql.database.windows.net;initial catalog=AdventureWorks;user id=sql-admin",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFEQVlGVUxMT0ZBREYtQURGXzA3MGU1MDIyLWEzYjMtNDFmMS1hOTk3LWU2ODE3Y2VhNjdhYyINCn0="
+		}
+	}
+}
@@ -0,0 +1,17 @@
+{
+	"name": "BatchForTraining01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBatch",
+		"typeProperties": {
+			"batchUri": "https://batchfortraining01.uksouth.batch.azure.com",
+			"poolName": "09a2ec47-0cc8-4492-935c-04852e3ec654",
+			"accountName": "batchfortraining01",
+			"linkedServiceName": {
+				"referenceName": "TrainingStore01",
+				"type": "LinkedServiceReference"
+			}
+		}
+	}
+}
@@ -0,0 +1,13 @@
+{
+	"name": "EmailSenderFunction",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureFunction",
+		"typeProperties": {
+			"functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net",
+			"authentication": "Anonymous",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9jZGYyMmI4Ni0zNTk2LTRkNzktYjdmOC1kNGNiMjUyNDZmYjUiDQp9"
+		}
+	}
+}
@@ -0,0 +1,16 @@
+{
+	"name": "GenericKeys",
+	"properties": {
+		"type": "AzureKeyVault",
+		"parameters": {
+			"baseUrl": {
+				"type": "String"
+			}
+		},
+		"annotations": [],
+		"typeProperties": {
+			"baseUrl": "@{linkedService().baseUrl}"
+		},
+		"description": "https://TrainingKeys01.vault.azure.net"
+	}
+}
@@ -0,0 +1,12 @@
+{
+	"name": "HTTP_AWGitHub",
+	"properties": {
+		"annotations": [],
+		"type": "HttpServer",
+		"typeProperties": {
+			"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/",
+			"enableServerCertificateValidation": true,
+			"authenticationType": "Anonymous"
+		}
+	}
+}
@@ -0,0 +1,12 @@
+{
+	"name": "HTTP_AWProduct",
+	"properties": {
+		"annotations": [],
+		"type": "HttpServer",
+		"typeProperties": {
+			"url": "https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/adventure-works/oltp-install-script/Product.csv",
+			"enableServerCertificateValidation": true,
+			"authenticationType": "Anonymous"
+		}
+	}
+}
@@ -0,0 +1,24 @@
+{
+	"name": "LaptopFiles",
+	"properties": {
+		"annotations": [],
+		"type": "FileServer",
+		"typeProperties": {
+			"host": "C:\\ADFRoot\\",
+			"userId": "ADFIRUser",
+			"password": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "ADFIRUser"
+			}
+		},
+		"connectVia": {
+			"referenceName": "Laptop",
+			"type": "IntegrationRuntimeReference"
+		}
+	},
+	"type": "Microsoft.DataFactory/factories/linkedservices"
+}
@@ -0,0 +1,11 @@
+{
+	"name": "TrainingKeys01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureKeyVault",
+		"typeProperties": {
+			"baseUrl": "https://TrainingKeys01.vault.azure.net/"
+		}
+	}
+}
@@ -0,0 +1,15 @@
+{
+	"name": "TrainingKeys01withUMI",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureKeyVault",
+		"typeProperties": {
+			"baseUrl": "https://TrainingKeys01.vault.azure.net/",
+			"credential": {
+				"referenceName": "TrainingCredential01",
+				"type": "CredentialReference"
+			}
+		}
+	}
+}
@@ -0,0 +1,12 @@
+{
+	"name": "TrainingStore01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobStorage",
+		"typeProperties": {
+			"connectionString": "DefaultEndpointsProtocol=https;AccountName=trainingstore01;EndpointSuffix=core.windows.net;",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMzMkUyRjlELTM1NzUtNDQ5Qy1BMEQyLTI0Qjg3Rjk5RUFFRV9hM2Y1NDEwZC01MzEzLTQ2MmItOTYwMC0yODdkNDE1NDM4MmEiDQp9"
+		}
+	}
+}
@@ -0,0 +1,18 @@
+{
+	"name": "trainingdb01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureSqlDatabase",
+		"typeProperties": {
+			"connectionString": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "ConnectionString-trainingdb01"
+			}
+		}
+	}
+}
@@ -0,0 +1,12 @@
+{
+	"name": "traininglak01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl82MDAxZmU4MS1mYzM3LTRjZjAtOWExNy00NTI2OWQ1MTJmZjIiDQp9"
+		}
+	}
+}
@@ -0,0 +1,19 @@
+{
+	"name": "traininglake01",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"accountKey": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "TrainingKeys01",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": "traininglake01"
+			}
+		}
+	}
+}
@@ -0,0 +1,12 @@
+{
+	"name": "traininglake01noneKV",
+	"type": "Microsoft.DataFactory/factories/linkedservices",
+	"properties": {
+		"annotations": [],
+		"type": "AzureBlobFS",
+		"typeProperties": {
+			"url": "https://traininglake01.dfs.core.windows.net",
+			"encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIlRSQUlOSU5HRkFDVE9SWURFVl8zYTM3NmM1OC0yOTQ4LTQwZWEtYjcwMC1lYjM0OWJmNzUyOTUiDQp9"
+		}
+	}
+}
@@ -0,0 +1,4 @@
+{
+	"name": "default",
+	"type": "Microsoft.DataFactory/factories/managedvirtualnetworks"
+}
@@ -0,0 +1,7 @@
+{
+	"name": "AzureFunctionTest",
+	"properties": {
+		"privateLinkResourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/ADF.procfwk/providers/Microsoft.Web/sites/FrameworkSupportFunctions",
+		"groupId": "sites"
+	}
+}
@@ -0,0 +1,10 @@
+{
+	"name": "AzureKeyVault1",
+	"properties": {
+		"privateLinkResourceId": "/subscriptions/450eaf4d-1124-4b6d-b490-95dedc991c1e/resourceGroups/Training/providers/Microsoft.KeyVault/vaults/TrainingKeys01",
+		"groupId": "vault",
+		"fqdns": [
+			"trainingkeys01.vaultcore.azure.net"
+		]
+	}
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Richard Swinbank	b8984ccca5	Revised labs	2023-05-31 09:09:28 +01:00
Paul Andrew	7d00f1dcde	Updating linkedService: LaptopFiles	2023-04-28 09:52:56 +01:00
Paul Andrew	9dc6566aee	Updating linkedService: LaptopFiles	2023-04-28 09:49:35 +01:00
Paul Andrew	a391fa37f5	Updating linkedService: LaptopFiles	2023-04-28 09:47:45 +01:00
Paul Andrew	05ea0eb291	Updating linkedService: LaptopFiles	2023-04-28 09:41:19 +01:00
Paul Andrew	db126971e2	Updating linkedService: LaptopFiles	2023-04-28 09:35:02 +01:00
Paul Andrew	f82936dbd2	Deleting integrationRuntime: PaulsLaptopIR	2023-04-22 16:56:17 +01:00
Paul Andrew	5955c39d4c	Updating linkedService: LaptopFiles	2023-04-22 11:11:46 +01:00
Paul Andrew	7c8b416d76	Deleting integrationRuntime: SSISIntegrationRuntime	2023-04-22 11:08:58 +01:00
Paul Andrew	c63bbf3fec	Adding integrationRuntime: Laptop	2023-04-22 11:06:36 +01:00
Paul Andrew	6bc8eb4bf2	Updating trigger: trigger1	2023-02-06 14:56:38 +00:00
Paul Andrew	1a773f1ea7	Updating trigger: trigger1	2023-02-06 14:56:27 +00:00
Paul Andrew	774c60a950	Adding trigger: trigger1	2023-02-06 14:56:17 +00:00
Paul Andrew	8da5f4407c	Adding managedPrivateEndpoint: AzureFunctionTest	2022-12-15 10:41:21 +00:00
Paul Andrew	83014f87ee	Deleting pipeline: pipeline1	2022-11-24 16:06:27 +00:00
Paul Andrew	20f78b5402	Adding pipeline: pipeline1	2022-10-28 12:36:33 +01:00
Paul Andrew	640adf9a43	Deleting pipeline: pipeline1	2022-10-28 11:28:27 +01:00
Paul Andrew	336183f053	Adding pipeline: pipeline1	2022-10-28 10:52:30 +01:00
Paul Andrew	19f49bd2ea	Updating pipeline: CopyAWUsingMetadata	2022-10-28 09:46:17 +01:00
Paul Andrew	8713b894a3	Updating integrationRuntime: PaulsLaptopIR	2022-10-27 21:19:06 +01:00
Richard Swinbank	d839132a0c	Merge pull request #2 from mrpaulandrewltd/copenhagen-labs2 Copenhagen labs2	2022-10-24 12:11:16 +01:00
Richard Swinbank	8b5b6e7815	Added lab 7 PDF	2022-10-24 12:08:33 +01:00
Paul Andrew	5e460f3eb4	Updating factory: TrainingFactoryDev	2022-10-24 11:45:17 +01:00
Richard Swinbank	767d751952	Updating pipeline: CopyAWUsingMetadata	2022-10-23 13:28:56 +01:00
Richard Swinbank	703a0dd035	updated catalog	2022-10-23 13:23:37 +01:00
Richard Swinbank	c42d769844	updated catalog	2022-10-23 13:18:28 +01:00
Richard Swinbank	15f214073b	updated catalog	2022-10-23 13:14:20 +01:00
Richard Swinbank	1da0cad168	updated catalog	2022-10-23 13:13:17 +01:00
Richard Swinbank	79ac0cf93c	updated catalog	2022-10-23 13:09:38 +01:00
Richard Swinbank	8734a1d506	Merge branch 'copenhagen-labs2' of https://github.com/mrpaulandrewltd/Azure-Data-Integration-Pipeline-Training into copenhagen-labs2	2022-10-23 13:06:16 +01:00
Richard Swinbank	0736afca98	updated catalog	2022-10-23 13:06:07 +01:00
Richard Swinbank	0324e85146	Updating pipeline: CopyAWUsingMetadata	2022-10-23 12:58:15 +01:00
Richard Swinbank	760dbc0ac2	updated catalog	2022-10-23 12:57:56 +01:00
Richard Swinbank	aa129c75a9	updated catalog	2022-10-23 12:53:18 +01:00
Richard Swinbank	10ff48ef1a	updated catalog	2022-10-23 12:43:35 +01:00
Richard Swinbank	1b048f3399	Merge branch 'main' into copenhagen-labs2	2022-10-22 18:13:13 +01:00
Richard Swinbank	d42694e0f8	Update TableCatalog.json	2022-10-22 18:07:32 +01:00
Richard Swinbank	7b8c94c517	Update TableCatalog.json	2022-10-22 18:03:30 +01:00
Richard Swinbank	b3a19db6c6	Update TableCatalog.json	2022-10-22 18:01:18 +01:00
Richard Swinbank	39a1ac99eb	Updating dataset: ADLS_PQT_AdventureWorks	2022-10-22 17:58:06 +01:00
Richard Swinbank	c194003994	Update TableCatalog.json	2022-10-22 17:55:19 +01:00
Richard Swinbank	8abd4355ce	Updating pipeline: CopyAWUsingMetadata	2022-10-22 17:52:19 +01:00
Richard Swinbank	5af7ca2526	Update TableCatalog.json	2022-10-22 17:51:45 +01:00
Richard Swinbank	c5269837bd	Updating pipeline: CopyAWUsingMetadata	2022-10-22 17:19:52 +01:00
Richard Swinbank	608fcaff97	Renaming pipeline: CopyAndMapAWEntity as CopyAWUsingMetadata	2022-10-22 17:19:49 +01:00
Richard Swinbank	8cb67f0240	Deleting linkedService: HTTP_GitHubUserContent	2022-10-22 17:17:24 +01:00
Richard Swinbank	4de3d227f6	Updating dataset: HTTP_TSV_AdventureWorks	2022-10-22 17:17:00 +01:00
Richard Swinbank	46836795c9	Updating pipeline: CopyAndMapAWEntity	2022-10-22 17:15:32 +01:00
Richard Swinbank	e59669bc66	Create TableCatalog.json	2022-10-22 17:14:28 +01:00
Richard Swinbank	6277a84161	Updating pipeline: CopyAndMapAWEntity	2022-10-22 17:09:20 +01:00
Richard Swinbank	047f7ffbfb	Updating pipeline: CopyAndMapAWEntity	2022-10-22 17:02:26 +01:00
Richard Swinbank	a8a73d3076	Updating pipeline: CopyAndMapAWEntity	2022-10-22 16:49:48 +01:00
Richard Swinbank	a9db253572	Updating pipeline: CopyAndMapAWEntity	2022-10-22 14:56:27 +01:00
Richard Swinbank	69875ce96c	Updating pipeline: CopyAndMapAWEntity	2022-10-22 14:50:26 +01:00
Richard Swinbank	7d85c298af	Updating pipeline: CopyAndMapAWEntity	2022-10-22 14:48:57 +01:00
Richard Swinbank	69e1b8a226	Updating pipeline: CopyAndMapAWEntity	2022-10-22 14:48:37 +01:00
Richard Swinbank	f58c20590e	Adding pipeline: CopyAndMapAWEntity	2022-10-22 14:46:19 +01:00
Richard Swinbank	2a1c8b20a9	Updating dataset: HTTP_TSV_AdventureWorks	2022-10-22 14:41:14 +01:00
Richard Swinbank	0b5ae59823	Adding dataset: ADLS_PQT_AdventureWorks	2022-10-22 14:39:52 +01:00
Richard Swinbank	410b5661c3	Updating dataset: HTTP_TSV_AdventureWorks	2022-10-22 14:36:46 +01:00
Richard Swinbank	0445cd075c	Updating dataset: HTTP_TSV_AdventureWorks	2022-10-22 14:35:04 +01:00
Richard Swinbank	ab4ec39d81	Adding dataset: HTTP_TSV_AdventureWorks	2022-10-22 14:34:09 +01:00
Richard Swinbank	afdeba1147	Adding linkedService: HTTP_GitHubUserContent	2022-10-22 14:31:17 +01:00
Richard Swinbank	6c6f070d0d	Updating dataset: ADLS_BIN_AWProduct Updating dataset: ADLS_BIN_AdventureWorks Updating dataset: ADLS_TSV_AdventureWorks Updating dataset: HTTP_BIN_AWProduct Updating dataset: HTTP_BIN_AdventureWorks	2022-10-22 14:27:04 +01:00
Richard Swinbank	e005e98b72	Updating pipeline: BuildDimProduct Updating pipeline: CopyAWEntities Updating pipeline: CopyAWEntity Updating pipeline: CopyAWProduct	2022-10-22 13:47:31 +01:00
Paul Andrew	3069f7e590	Updating dataflow: MappingOrderAggregationWithParam	2022-10-22 11:44:34 +01:00
Paul Andrew	f6b2bde9cb	Adding pipeline: Order Summary with Mapping With Param	2022-10-22 11:39:05 +01:00
Paul Andrew	3bb9e481e9	Adding dataflow: MappingOrderAggregationWithParam	2022-10-22 11:36:33 +01:00
Paul Andrew	aee4f87edc	Updating pipeline: EmailSender	2022-10-17 12:48:05 +01:00
Paul Andrew	918640b092	Updating pipeline: EmailSender	2022-10-17 12:47:15 +01:00
Paul Andrew	a5fe582f55	Adding pipeline: EmailSender	2022-10-17 12:43:30 +01:00
Paul Andrew	18b10c5bab	Adding linkedService: EmailSenderFunction	2022-10-17 12:41:00 +01:00
Paul Andrew	67dc7b9739	Updating pipeline: Custom Activity	2022-10-10 09:47:08 +01:00
Paul Andrew	15f4f40a00	Updating pipeline: 1950 Activities	2022-09-01 12:44:38 +01:00
Paul Andrew	2d17b05833	Demo run code tweaks	2022-08-30 16:44:39 +01:00
Paul Andrew	5bbe79c199	Updating pipeline: Wait 10	2022-08-30 16:22:56 +01:00
Paul Andrew	401aa681d8	Updating pipeline: Custom Activity	2022-08-30 16:07:35 +01:00
Paul Andrew	f16c8708ff	Updating pipeline: 06 - Example Global Param	2022-08-30 15:32:41 +01:00
Paul Andrew	08c36321ad	Data Scotland - Final Updates	2022-08-30 11:43:55 +01:00
Richard Swinbank	8920a4857c	Merge pull request #1 from mrpaulandrewltd/lab-revisions Lab revisions	2022-08-30 11:15:01 +01:00
Richard Swinbank	b82e31bf3a	Updating linkedService: ADLS_saintegrationpipelines	2022-08-30 11:10:52 +01:00
Richard Swinbank	c2b7d903cb	Added lab ADF artifacts	2022-08-29 21:37:11 +01:00
Richard Swinbank	213168013b	Removed factory artifacts	2022-08-29 21:32:02 +01:00
Richard Swinbank	8115d634b9	Replaced PDFs	2022-08-29 21:24:14 +01:00
Paul Andrew	bb11a6056a	Update README.md	2022-08-18 16:33:29 +01:00
Paul Andrew	149ccc3b1a	VS tweaks	2022-08-16 12:01:49 +01:00
Paul Andrew	8f0cc9bb32	PDF Updates	2022-08-16 10:29:56 +01:00
Paul Andrew	4cf0d00eb0	Fix broken links	2022-08-16 10:26:55 +01:00
Paul Andrew	5262dd5fc0	Update 02. Agenda.md	2022-08-16 10:25:00 +01:00
Paul Andrew	dca62f67e7	Repo reorg	2022-08-16 08:42:57 +01:00
Paul Andrew	d969bf2580	Tweaks	2022-04-21 13:36:38 +01:00
Paul Andrew	40f164f38e	Folder restructure	2022-04-21 13:27:19 +01:00
Paul Andrew	3c25c7a724	Relative path updates	2022-04-21 12:56:41 +01:00
Paul Andrew	26af931aed	Relative path updates	2022-04-21 12:54:11 +01:00
Paul Andrew	efc1e517f8	Test update	2022-04-21 12:51:38 +01:00
Paul Andrew	4a9384eb91	Repo refactor part 1	2022-04-21 12:36:52 +01:00
Paul Andrew	c404b6c863	Update publish_config.json	2022-03-25 15:23:12 +00:00
Paul Andrew	024c3f8b8d	Deleting pipeline: pipeline1	2022-03-16 16:37:30 +00:00
Paul Andrew	067fa92909	Update README.md	2022-03-16 16:35:21 +00:00
Paul Andrew	09c848bc7d	Updating pipeline: pipeline1	2022-03-09 15:58:17 +00:00
Paul Andrew	c168fb875f	Adding pipeline: pipeline1	2022-03-09 15:34:21 +00:00
Paul Andrew	2139653544	SQLBits Prep	2022-03-07 10:31:10 +00:00
Paul Andrew	fd50f773a1	Adding templates: 04 - Upload - From Metadata/manifest Adding templates: 04 - Upload - From Metadata/04 - Upload - From Metadata	2022-03-01 14:15:21 +00:00
Paul Andrew	2f415a56fc	Updating pipeline: Wait 10	2022-03-01 13:11:58 +00:00
Paul Andrew	b24737c787	Deleting pipeline: pipeline1	2022-03-01 12:42:34 +00:00
Paul Andrew	37945fadde	Updating pipeline: pipeline1	2022-03-01 12:41:45 +00:00
Paul Andrew	96a37399d0	Updating pipeline: pipeline1	2022-03-01 12:40:28 +00:00
Paul Andrew	76244d015b	Adding pipeline: pipeline1	2022-03-01 12:39:40 +00:00
Paul Andrew	cd0b49601a	Day 2 pdfs	2022-03-01 09:16:42 +00:00
Paul Andrew	87fb2ae758	Updating pipeline: Wait 10	2022-02-28 20:13:02 +00:00
Paul Andrew	c281e3cedf	Renaming pipeline: Wait 15 as Wait 10	2022-02-28 20:13:00 +00:00
Paul Andrew	6cfb707496	Updating pipeline: Wait 15	2022-02-28 20:10:11 +00:00
Paul Andrew	5331637e5b	Updating pipeline: Wait 15	2022-02-28 20:09:52 +00:00
Paul Andrew	3d4f654f49	Renaming pipeline: Wait 10 as Wait 15	2022-02-28 20:09:50 +00:00
Paul Andrew	e6ef95d3cf	Deleting pipeline: pipeline1	2022-02-28 16:28:43 +00:00
Paul Andrew	5cd128a83a	Adding pipeline: pipeline1	2022-02-28 16:22:21 +00:00
Paul Andrew	e4d1d5a808	Deleting dataset: AzureSqlTable1	2022-02-28 16:17:36 +00:00
Paul Andrew	416786c820	Deleting pipeline: pipeline1	2022-02-28 16:17:29 +00:00
Paul Andrew	0878a36383	Adding dataset: AzureSqlTable1	2022-02-28 16:11:16 +00:00
Paul Andrew	48b99cc403	Adding pipeline: pipeline1	2022-02-28 16:02:45 +00:00
Paul Andrew	3439868f62	Various updates	2022-02-28 07:37:47 +00:00
Paul Andrew	5868d4ed76	Code restructure	2022-02-26 20:14:43 +00:00
Paul Andrew	8d07f3c5d3	Updating pipeline: 04 - Upload - From Metadata	2022-02-26 10:38:08 +00:00
Paul Andrew	25a0c81d5e	Updating pipeline: 04 - Upload - From Metadata	2022-02-26 10:30:35 +00:00
Paul Andrew	101c565807	Adding dataset: LaptopFilesParquet	2022-02-26 10:29:39 +00:00
Paul Andrew	8224239a4c	Updating pipeline: 04 - Upload - From Metadata	2022-02-26 10:27:06 +00:00
Paul Andrew	4782881321	Updating pipeline: 04 - Upload - From Metadata	2022-02-26 10:26:03 +00:00
Paul Andrew	63cbbcd637	Updating pipeline: 04 - Upload - From Metadata	2022-02-26 10:23:22 +00:00
Paul Andrew	ce14b714ef	Updating pipeline: 04 - Upload - From Metadata Adding dataset: LaptopFilesBinary Updating dataset: LakeFilesBinary	2022-02-26 10:18:40 +00:00
Paul Andrew	a14d4e5fcd	Adding dataset: LakeFilesBinary	2022-02-26 10:17:31 +00:00
Paul Andrew	16765e4920	Updating integrationRuntime: PaulsLaptopIR	2022-02-26 09:52:47 +00:00
Paul Andrew	c491e50f97	Update README.md	2022-02-23 14:38:23 +00:00
Paul Andrew	e60792efc6	Updating pipeline: Get Key Vault Value	2022-02-23 14:10:21 +00:00
Paul Andrew	6f97cfa210	Updating pipeline: Get Key Vault Value	2022-02-23 14:08:46 +00:00
Paul Andrew	9bc4f2c8fa	Adding dataset: AnyDatabaseTableAnyKeyVault	2022-02-23 14:04:21 +00:00
Paul Andrew	d77d3ce6bd	Adding linkedService: AnyDatabaseConnectionFromAnyKeyVault	2022-02-23 14:02:18 +00:00
Paul Andrew	6000b21f1e	Updating pipeline: Get Key Vault Value	2022-02-23 13:56:48 +00:00
Paul Andrew	6940fb6b84	Updating factory: TrainingFactoryDev	2022-02-23 13:56:12 +00:00
Paul Andrew	b950d6cfa8	Updating pipeline: Get Key Vault Value	2022-02-23 13:41:55 +00:00
Paul Andrew	00f04f2425	Adding pipeline: Get Key Vault Value	2022-02-23 13:35:07 +00:00
Paul Andrew	162463a963	Deleting linkedService: AzureSqlDatabase1	2022-02-22 10:47:44 +00:00
Paul Andrew	64fd3b8f89	Adding linkedService: AzureSqlDatabase1	2022-02-22 10:45:49 +00:00
Paul Andrew	4e54af8764	Deleting linkedService: AzureKeyVault1	2022-02-22 10:43:29 +00:00
Paul Andrew	bf5c6839fd	Adding linkedService: AzureKeyVault1	2022-02-22 10:42:52 +00:00
Paul Andrew	a364737a99	Updating linkedService: GenericKeys	2022-02-22 10:41:22 +00:00
Paul Andrew	590fca7d2d	Adding linkedService: GenericKeys	2022-02-22 10:40:19 +00:00
Paul Andrew	4114355f08	Updating pipeline: 06 - Example Global Param	2022-02-22 10:13:22 +00:00
Paul Andrew	f9f7b08681	Updating pipeline: 06 - Example Global Param	2022-02-22 09:38:03 +00:00
Paul Andrew	c899656fb8	Renaming pipeline: pipeline1 as 06 - Example Global Param	2022-02-22 09:38:02 +00:00
Paul Andrew	3a4c88bcf8	Updating pipeline: pipeline1	2022-02-22 09:36:38 +00:00
Paul Andrew	807683348b	Updating factory: TrainingFactoryDev	2022-02-22 09:33:51 +00:00
Paul Andrew	968ae66b93	Updating pipeline: pipeline1	2022-02-22 09:21:14 +00:00
Paul Andrew	24fc0e9d37	Updating pipeline: pipeline1	2022-02-22 09:19:42 +00:00
Paul Andrew	f81a37adc4	Adding pipeline: 05 - Get File List Utility	2022-02-22 09:19:29 +00:00
Paul Andrew	32de076603	Adding pipeline: pipeline1	2022-02-22 09:16:27 +00:00
Paul Andrew	e443c98329	Adding pipeline: Wait 10	2022-02-22 09:16:10 +00:00
Paul Andrew	291d4dad0e	Removed design files	2022-02-11 11:25:51 +00:00
Paul Andrew	3490273d3d	Readme Update	2022-02-11 11:09:17 +00:00
Paul Andrew	faa1043253	Create FUNDING.yml	2022-02-11 11:05:00 +00:00
Paul Andrew	2249e58e5c	Update README.md	2022-02-11 10:50:26 +00:00
Paul Andrew	beb9dbf086	Create Contact.png	2022-02-11 10:50:16 +00:00
Paul Andrew	11ce675fac	Structure Overhaul	2022-02-11 10:31:23 +00:00
Paul Andrew	bf9ceef777	Create publish_config.json	2022-02-11 10:15:04 +00:00
Paul Andrew	be14d332c0	Adding credential: TrainingCredential01 Adding dataflow: MappingOrderAggregation Adding dataset: LakeFileOrderHeaderParquet Adding dataset: LakeFileOrderDetailLinesParquet Adding dataset: TableOrderSummary Adding linkedService: traininglake01noneKV Adding linkedService: trainingdb01 Adding linkedService: TrainingKeys01 Adding dataflow: WranglingOrderAggregation Adding dataset: GetSetMetadata Adding dataset: AnyDatabaseTable Adding linkedService: AnyDatabaseConnection Adding dataset: LaptopFolders Adding linkedService: LaptopFiles Adding integrationRuntime: PaulsLaptopIR Adding dataset: LaptopFilePersonCSV Adding dataset: LakeFiles Adding linkedService: traininglake01 Adding dataset: LaptopsFiles Adding dataset: LakeFilePersonCSV Adding dataset: LakeFileParquet Adding integrationRuntime: ForDataFlowDemos Adding integrationRuntime: VNetEnabledIR Adding managedVirtualNetwork: default Adding linkedService: traininglak01 Adding linkedService: TrainingKeys01withUMI Adding linkedService: TrainingStore01 Adding linkedService: BatchForTraining01 Adding pipeline: Scale Out Level 2 Adding pipeline: Scale Out Level 1 Adding pipeline: Lazy Replication Adding pipeline: 03 - Upload - From Discovery Adding pipeline: 01 - Upload - Simple Adding pipeline: 04 - Upload - From Metadata Adding pipeline: Order Summary with Mapping Adding pipeline: 02 - Upload - Copy Params Adding pipeline: 1950 Activities Adding pipeline: Custom Activity Adding managedPrivateEndpoint: AzureKeyVault1 Adding factory: TrainingFactoryDev	2022-02-11 10:15:02 +00:00
Paul Andrew	21254cf18e	marketing tweaks	2022-01-24 15:59:51 +00:00
Paul Andrew	22bc0a70df	Marketing bits	2022-01-21 11:16:47 +00:00
Paul Andrew	cb27d25190	Initial commit	2022-01-14 11:34:46 +00:00