Repo reorg

Code/CustomActivity/obj/CustomActivity.csproj.nuget.dgspec.json

@@ -1,17 +1,17 @@
 {
   "format": 1,
   "restore": {
-    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj": {}
+    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {}
   },
   "projects": {
-    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj": {
+    "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj": {
       "version": "1.0.0",
       "restore": {
-        "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj",
+        "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
         "projectName": "CustomActivity",
-        "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj",
+        "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
         "packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
-        "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\obj\\",
+        "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
         "projectStyle": "PackageReference",
         "fallbackFolders": [
           "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
@@ -58,7 +58,7 @@
               "privateAssets": "all"
             }
           },
-          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.405\\RuntimeIdentifierGraph.json"
+          "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
         }
       }
     }

Code/CustomActivity/obj/Debug/netcoreapp3.1/CustomActivity.GeneratedMSBuildEditorConfig.editorconfig

@@ -1,3 +1,3 @@
 is_global = true
 build_property.RootNamespace = CustomActivity
-build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\Code\CustomActivity\
+build_property.ProjectDir = C:\Users\paul.andrew\GitHub\Azure-Data-Integration-Pipeline-Training\Code\CustomActivity\

Code/CustomActivity/obj/project.assets.json

@@ -14,11 +14,11 @@
   "project": {
     "version": "1.0.0",
     "restore": {
-      "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj",
+      "projectUniqueName": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
       "projectName": "CustomActivity",
-      "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj",
+      "projectPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
       "packagesPath": "C:\\Users\\paul.andrew\\.nuget\\packages\\",
-      "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\obj\\",
+      "outputPath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\obj\\",
       "projectStyle": "PackageReference",
       "fallbackFolders": [
         "C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
@@ -65,7 +65,7 @@
             "privateAssets": "all"
           }
         },
-        "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.405\\RuntimeIdentifierGraph.json"
+        "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\5.0.408\\RuntimeIdentifierGraph.json"
       }
     }
   }

Code/CustomActivity/obj/project.nuget.cache

@@ -1,8 +1,8 @@
 {
   "version": 2,
-  "dgSpecHash": "xRYOk9IhudNquJUU9b9JCl1B1oiWukwfQcF3G+uz+Yc2wIvOQgSgEa86Jepb1nvUAUel2waPl+qrCo11gOh5Lg==",
+  "dgSpecHash": "f0p4CwgurPXk2v0yD5sG6nLAvje2rr5wHxFdncruLvsOGiK3Q2I9TnSpBC64Z+fnkevGX0xQTI2XPAbWzSUW3A==",
   "success": true,
-  "projectFilePath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipelines-Advanced-Design-and-Delivery\\Code\\CustomActivity\\CustomActivity.csproj",
+  "projectFilePath": "C:\\Users\\paul.andrew\\GitHub\\Azure-Data-Integration-Pipeline-Training\\Code\\CustomActivity\\CustomActivity.csproj",
   "expectedPackageFiles": [],
   "logs": []
 }

Content/02. Fundamentals to Level 300 - 1 Day/02. Agenda.md

@@ -2,12 +2,21 @@
 ## Fundamentals to Level 300
 ### 1 Day
 
-![Slide Header](./02.%20Agenda.md)
+![Slide Header](./02.%20Agenda%20Header.png)
 
 ___
 
 ## Overview
 
+In this full day of training, we’ll start with the very basics and learn how to orchestrate your Azure data platform from start to finish. You will learn how to build out Azure control flow and data flow components as processing pipelines using Azure Data Factory and Azure Synapse Analytics. We’ll start by covering the fundamentals within the resources and together build out pipelines that ingest data from local source systems, transform and serve it to consumers. We’ll then continue taking an end-to-end look at our Azure integration pipeline tools within highly scalable cloud native architectures, dealing with triggering, monitoring, dynamic pipeline content as well as CI/CD practices. Start the day knowing nothing about Azure Data Integration pipelines and leave with the knowledge, slides, demos, and code to apply these resources in your role as a data engineering professional.
+___
+
+## Milestones
+
+* Pipelines - the fundamentals
+* Getting hands with the basics
+* Pipelines - advanced features & extensibility
+* Pipelines - real world implementation
 
 ___
 
@@ -15,6 +24,101 @@ ___
 
 The following offers an insight into the complete agenda and module breakdown for this workshop.
 
-
+* __Module 1:__ [Pipeline Fundamentals](./Module%201.pdf)
+  * The History of Azure Orchestration
+  * Synapse Analytics vs Data Factory
+  * Integration Components
+  * Common Activities
+  * Execution Dependencies
 
 ___
+
+* __Module 2:__ [Integration Runtime Design Patterns](./Module%202.pdf)
+  * Compute Types
+    * Azure
+    * Hosted
+    * SSIS
+  * Patterns & Configuration
+
+___
+
+* __Module 3:__ [Data Transformation](./Module%203.pdf)
+  * Data Flows
+  * Power Query Injection
+  * Spark Configuration
+  * Use Cases
+
+___
+
+* __Module 4:__ [Dynamic Pipelines](./Module%204.pdf)
+  * Expressions & Interpolation
+  * Simple Metadata Driven Execution
+  * Dynamic Content Chains
+  * Reference Names
+
+___
+
+* __Module 5:__ [Pipeline Extensibility](./Module%205.pdf)
+  * Azure Batch Service
+    * Tasks
+    * Compute Pools
+    * Scaling
+  * Pipeline Custom Activities
+  * Azure Management API
+  * Azure Functions 
+
+___
+
+* __Module 6:__ [Execution Parallelism](./Module%206.pdf)
+  * Control Flow Scale Out
+  * Concurrency Limitations
+  * Internal vs External Activities
+  * Orchestration Framework - [procfwk.com](http://procfwk.com/)
+
+___
+
+* __Module 7:__ [VNet Integration](./Module%207.pdf)
+  * Private Endpoints
+  * Managed VNet's
+  * Firewall Bypass
+
+___
+
+* __Module 8:__ [Security](./Module%208.pdf)
+  * Service Principals
+  * Managed Identities
+  * Azure Key Vault Integration
+  * Customer Managed Keys
+  * Pipeline Access & Permissions
+
+___
+
+* __Module 9:__ [Monitoring & Alerting](./Module%209.pdf)
+    * Studio Monitoring
+    * Log Analytics & Kusto Queries
+    * Operational Dashboards
+    * Advanced Alerting
+
+___
+
+* __Module 10:__ [Solution Testing](./Module%2010.pdf)
+    * Development Time Validation
+    * Test Coverage
+    * NUnit Tests
+
+___
+
+* __Module 11:__ [CI/CD](./Module%2011.pdf)
+    * Source Control vs Developer UI
+    * Basic ARM Template Deployments
+    * Advanced Deployment Patterns
+
+___
+
+* __Module 12:__ [Final Thoughts](./Module%2012.pdf)
+  * Running Costs
+  * Conclusions
+  * Best Practices
+
+
+___

Labs/vs/.gitignore

@@ -1,362 +0,0 @@
-## Ignore Visual Studio temporary files, build results, and
-## files generated by popular Visual Studio add-ons.
-##
-## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
-
-# User-specific files
-*.rsuser
-*.suo
-*.user
-*.userosscache
-*.sln.docstates
-
-# User-specific files (MonoDevelop/Xamarin Studio)
-*.userprefs
-
-# Mono auto generated files
-mono_crash.*
-
-# Build results
-[Dd]ebug/
-[Dd]ebugPublic/
-[Rr]elease/
-[Rr]eleases/
-x64/
-x86/
-[Ww][Ii][Nn]32/
-[Aa][Rr][Mm]/
-[Aa][Rr][Mm]64/
-bld/
-[Bb]in/
-[Oo]bj/
-[Ll]og/
-[Ll]ogs/
-
-# Visual Studio 2015/2017 cache/options directory
-.vs/
-# Uncomment if you have tasks that create the project's static files in wwwroot
-#wwwroot/
-
-# Visual Studio 2017 auto generated files
-Generated\ Files/
-
-# MSTest test Results
-[Tt]est[Rr]esult*/
-[Bb]uild[Ll]og.*
-
-# NUnit
-*.VisualState.xml
-TestResult.xml
-nunit-*.xml
-
-# Build Results of an ATL Project
-[Dd]ebugPS/
-[Rr]eleasePS/
-dlldata.c
-
-# Benchmark Results
-BenchmarkDotNet.Artifacts/
-
-# .NET Core
-project.lock.json
-project.fragment.lock.json
-artifacts/
-
-# ASP.NET Scaffolding
-ScaffoldingReadMe.txt
-
-# StyleCop
-StyleCopReport.xml
-
-# Files built by Visual Studio
-*_i.c
-*_p.c
-*_h.h
-*.ilk
-*.meta
-*.obj
-*.iobj
-*.pch
-*.pdb
-*.ipdb
-*.pgc
-*.pgd
-*.rsp
-*.sbr
-*.tlb
-*.tli
-*.tlh
-*.tmp
-*.tmp_proj
-*_wpftmp.csproj
-*.log
-*.vspscc
-*.vssscc
-.builds
-*.pidb
-*.svclog
-*.scc
-
-# Chutzpah Test files
-_Chutzpah*
-
-# Visual C++ cache files
-ipch/
-*.aps
-*.ncb
-*.opendb
-*.opensdf
-*.sdf
-*.cachefile
-*.VC.db
-*.VC.VC.opendb
-
-# Visual Studio profiler
-*.psess
-*.vsp
-*.vspx
-*.sap
-
-# Visual Studio Trace Files
-*.e2e
-
-# TFS 2012 Local Workspace
-$tf/
-
-# Guidance Automation Toolkit
-*.gpState
-
-# ReSharper is a .NET coding add-in
-_ReSharper*/
-*.[Rr]e[Ss]harper
-*.DotSettings.user
-
-# TeamCity is a build add-in
-_TeamCity*
-
-# DotCover is a Code Coverage Tool
-*.dotCover
-
-# AxoCover is a Code Coverage Tool
-.axoCover/*
-!.axoCover/settings.json
-
-# Coverlet is a free, cross platform Code Coverage Tool
-coverage*.json
-coverage*.xml
-coverage*.info
-
-# Visual Studio code coverage results
-*.coverage
-*.coveragexml
-
-# NCrunch
-_NCrunch_*
-.*crunch*.local.xml
-nCrunchTemp_*
-
-# MightyMoose
-*.mm.*
-AutoTest.Net/
-
-# Web workbench (sass)
-.sass-cache/
-
-# Installshield output folder
-[Ee]xpress/
-
-# DocProject is a documentation generator add-in
-DocProject/buildhelp/
-DocProject/Help/*.HxT
-DocProject/Help/*.HxC
-DocProject/Help/*.hhc
-DocProject/Help/*.hhk
-DocProject/Help/*.hhp
-DocProject/Help/Html2
-DocProject/Help/html
-
-# Click-Once directory
-publish/
-
-# Publish Web Output
-*.[Pp]ublish.xml
-*.azurePubxml
-# Note: Comment the next line if you want to checkin your web deploy settings,
-# but database connection strings (with potential passwords) will be unencrypted
-*.pubxml
-*.publishproj
-
-# Microsoft Azure Web App publish settings. Comment the next line if you want to
-# checkin your Azure Web App publish settings, but sensitive information contained
-# in these scripts will be unencrypted
-PublishScripts/
-
-# NuGet Packages
-*.nupkg
-# NuGet Symbol Packages
-*.snupkg
-# The packages folder can be ignored because of Package Restore
-**/[Pp]ackages/*
-# except build/, which is used as an MSBuild target.
-!**/[Pp]ackages/build/
-# Uncomment if necessary however generally it will be regenerated when needed
-#!**/[Pp]ackages/repositories.config
-# NuGet v3's project.json files produces more ignorable files
-*.nuget.props
-*.nuget.targets
-
-# Microsoft Azure Build Output
-csx/
-*.build.csdef
-
-# Microsoft Azure Emulator
-ecf/
-rcf/
-
-# Windows Store app package directories and files
-AppPackages/
-BundleArtifacts/
-Package.StoreAssociation.xml
-_pkginfo.txt
-*.appx
-*.appxbundle
-*.appxupload
-
-# Visual Studio cache files
-# files ending in .cache can be ignored
-*.[Cc]ache
-# but keep track of directories ending in .cache
-!?*.[Cc]ache/
-
-# Others
-ClientBin/
-~$*
-*~
-*.dbmdl
-*.dbproj.schemaview
-*.jfm
-*.pfx
-*.publishsettings
-orleans.codegen.cs
-
-# Including strong name files can present a security risk
-# (https://github.com/github/gitignore/pull/2483#issue-259490424)
-#*.snk
-
-# Since there are multiple workflows, uncomment next line to ignore bower_components
-# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
-#bower_components/
-
-# RIA/Silverlight projects
-Generated_Code/
-
-# Backup & report files from converting an old project file
-# to a newer Visual Studio version. Backup files are not needed,
-# because we have git ;-)
-_UpgradeReport_Files/
-Backup*/
-UpgradeLog*.XML
-UpgradeLog*.htm
-ServiceFabricBackup/
-*.rptproj.bak
-
-# SQL Server files
-*.mdf
-*.ldf
-*.ndf
-
-# Business Intelligence projects
-*.rdl.data
-*.bim.layout
-*.bim_*.settings
-*.rptproj.rsuser
-*- [Bb]ackup.rdl
-*- [Bb]ackup ([0-9]).rdl
-*- [Bb]ackup ([0-9][0-9]).rdl
-
-# Microsoft Fakes
-FakesAssemblies/
-
-# GhostDoc plugin setting file
-*.GhostDoc.xml
-
-# Node.js Tools for Visual Studio
-.ntvs_analysis.dat
-node_modules/
-
-# Visual Studio 6 build log
-*.plg
-
-# Visual Studio 6 workspace options file
-*.opt
-
-# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
-*.vbw
-
-# Visual Studio LightSwitch build output
-**/*.HTMLClient/GeneratedArtifacts
-**/*.DesktopClient/GeneratedArtifacts
-**/*.DesktopClient/ModelManifest.xml
-**/*.Server/GeneratedArtifacts
-**/*.Server/ModelManifest.xml
-_Pvt_Extensions
-
-# Paket dependency manager
-.paket/paket.exe
-paket-files/
-
-# FAKE - F# Make
-.fake/
-
-# CodeRush personal settings
-.cr/personal
-
-# Python Tools for Visual Studio (PTVS)
-__pycache__/
-*.pyc
-
-# Cake - Uncomment if you are using it
-# tools/**
-# !tools/packages.config
-
-# Tabs Studio
-*.tss
-
-# Telerik's JustMock configuration file
-*.jmconfig
-
-# BizTalk build output
-*.btp.cs
-*.btm.cs
-*.odx.cs
-*.xsd.cs
-
-# OpenCover UI analysis results
-OpenCover/
-
-# Azure Stream Analytics local run output
-ASALocalRun/
-
-# MSBuild Binary and Structured Log
-*.binlog
-
-# NVidia Nsight GPU debugger configuration file
-*.nvuser
-
-# MFractors (Xamarin productivity tool) working folder
-.mfractor/
-
-# Local History for Visual Studio
-.localhistory/
-
-# BeatPulse healthcheck temp database
-healthchecksdb
-
-# Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
-
-# Ionide (cross platform F# VS Code tools) working folder
-.ionide/
-
-# Fody - auto-generated XML schema
-FodyWeavers.xsd