{
	"name": "04 - Upload - From Metadata",
	"properties": {
		"activities": [
			{
				"name": "Get File List",
				"type": "Lookup",
				"dependsOn": [],
				"policy": {
					"timeout": "7.00:00:00",
					"retry": 0,
					"retryIntervalInSeconds": 30,
					"secureOutput": false,
					"secureInput": false
				},
				"userProperties": [],
				"typeProperties": {
					"source": {
						"type": "AzureSqlSource",
						"sqlReaderQuery": "SELECT * FROM [dbo].[FilesToUpload] WHERE [Enabled] = 1;",
						"queryTimeout": "02:00:00",
						"partitionOption": "None"
					},
					"dataset": {
						"referenceName": "GetSetMetadata",
						"type": "DatasetReference"
					},
					"firstRowOnly": false
				}
			},
			{
				"name": "UploadFiles",
				"type": "ForEach",
				"dependsOn": [
					{
						"activity": "Get File List",
						"dependencyConditions": [
							"Succeeded"
						]
					},
					{
						"activity": "Log Upload Start",
						"dependencyConditions": [
							"Succeeded"
						]
					}
				],
				"userProperties": [],
				"typeProperties": {
					"items": {
						"value": "@activity('Get File List').output.value",
						"type": "Expression"
					},
					"activities": [
						{
							"name": "Copy By File Type",
							"type": "Switch",
							"dependsOn": [],
							"userProperties": [],
							"typeProperties": {
								"on": {
									"value": "@substring(item().FileName,add(indexof(item().FileName,'.'),1),sub(length(item().FileName),add(indexof(item().FileName,'.'),1)))",
									"type": "Expression"
								},
								"cases": [
									{
										"value": "csv",
										"activities": [
											{
												"name": "CSV Copy",
												"type": "Copy",
												"dependsOn": [],
												"policy": {
													"timeout": "7.00:00:00",
													"retry": 0,
													"retryIntervalInSeconds": 30,
													"secureOutput": false,
													"secureInput": false
												},
												"userProperties": [],
												"typeProperties": {
													"source": {
														"type": "DelimitedTextSource",
														"additionalColumns": [
															{
																"name": "FilePath",
																"value": "$$FILEPATH"
															},
															{
																"name": "Environment",
																"value": {
																	"value": "@pipeline().globalParameters.Environment",
																	"type": "Expression"
																}
															},
															{
																"name": "PipelineName",
																"value": {
																	"value": "@pipeline().Pipeline",
																	"type": "Expression"
																}
															},
															{
																"name": "RunId",
																"value": {
																	"value": "@pipeline().RunId",
																	"type": "Expression"
																}
															},
															{
																"name": "RunDate",
																"value": {
																	"value": "@utcnow()",
																	"type": "Expression"
																}
															}
														],
														"storeSettings": {
															"type": "FileServerReadSettings",
															"recursive": false,
															"enablePartitionDiscovery": false
														},
														"formatSettings": {
															"type": "DelimitedTextReadSettings"
														}
													},
													"sink": {
														"type": "DelimitedTextSink",
														"storeSettings": {
															"type": "AzureBlobFSWriteSettings"
														},
														"formatSettings": {
															"type": "DelimitedTextWriteSettings",
															"quoteAllText": true,
															"fileExtension": ".txt"
														}
													},
													"enableStaging": false,
													"translator": {
														"type": "TabularTranslator",
														"typeConversion": true,
														"typeConversionSettings": {
															"allowDataTruncation": true,
															"treatBooleanAsNumber": false
														}
													}
												},
												"inputs": [
													{
														"referenceName": "LaptopsFiles",
														"type": "DatasetReference",
														"parameters": {
															"Directory": {
																"value": "@item().SourceDirectory",
																"type": "Expression"
															},
															"File": {
																"value": "@item().FileName",
																"type": "Expression"
															}
														}
													}
												],
												"outputs": [
													{
														"referenceName": "LakeFiles",
														"type": "DatasetReference",
														"parameters": {
															"Directory": {
																"value": "@item().TargetDirectory",
																"type": "Expression"
															},
															"File": {
																"value": "@item().FileName",
																"type": "Expression"
															}
														}
													}
												]
											}
										]
									},
									{
										"value": "parquet",
										"activities": [
											{
												"name": "Parquet Copy",
												"type": "Copy",
												"dependsOn": [],
												"policy": {
													"timeout": "7.00:00:00",
													"retry": 0,
													"retryIntervalInSeconds": 30,
													"secureOutput": false,
													"secureInput": false
												},
												"userProperties": [],
												"typeProperties": {
													"source": {
														"type": "ParquetSource",
														"storeSettings": {
															"type": "FileServerReadSettings",
															"recursive": false,
															"enablePartitionDiscovery": false
														}
													},
													"sink": {
														"type": "ParquetSink",
														"storeSettings": {
															"type": "AzureBlobFSWriteSettings"
														},
														"formatSettings": {
															"type": "ParquetWriteSettings"
														}
													},
													"enableStaging": false,
													"translator": {
														"type": "TabularTranslator",
														"typeConversion": true,
														"typeConversionSettings": {
															"allowDataTruncation": true,
															"treatBooleanAsNumber": false
														}
													}
												},
												"inputs": [
													{
														"referenceName": "LaptopFilesParquet",
														"type": "DatasetReference",
														"parameters": {
															"Directory": {
																"value": "@item().SourceDirectory",
																"type": "Expression"
															},
															"File": {
																"value": "@item().FileName",
																"type": "Expression"
															}
														}
													}
												],
												"outputs": [
													{
														"referenceName": "LakeFileParquet",
														"type": "DatasetReference",
														"parameters": {
															"Directory": {
																"value": "@item().TargetDirectory",
																"type": "Expression"
															},
															"File": {
																"value": "@item().FileName",
																"type": "Expression"
															}
														}
													}
												]
											}
										]
									}
								],
								"defaultActivities": [
									{
										"name": "Binary Copy",
										"type": "Copy",
										"dependsOn": [],
										"policy": {
											"timeout": "7.00:00:00",
											"retry": 0,
											"retryIntervalInSeconds": 30,
											"secureOutput": false,
											"secureInput": false
										},
										"userProperties": [],
										"typeProperties": {
											"source": {
												"type": "BinarySource",
												"storeSettings": {
													"type": "FileServerReadSettings",
													"recursive": false
												},
												"formatSettings": {
													"type": "BinaryReadSettings"
												}
											},
											"sink": {
												"type": "BinarySink",
												"storeSettings": {
													"type": "AzureBlobFSWriteSettings"
												}
											},
											"enableStaging": false
										},
										"inputs": [
											{
												"referenceName": "LaptopFilesBinary",
												"type": "DatasetReference",
												"parameters": {
													"Directory": {
														"value": "@item().SourceDirectory",
														"type": "Expression"
													},
													"File": {
														"value": "@item().FileName",
														"type": "Expression"
													}
												}
											}
										],
										"outputs": [
											{
												"referenceName": "LakeFilesBinary",
												"type": "DatasetReference",
												"parameters": {
													"Directory": {
														"value": "@item().TargetDirectory",
														"type": "Expression"
													},
													"File": {
														"value": "@item().FileName",
														"type": "Expression"
													}
												}
											}
										]
									}
								]
							}
						}
					]
				}
			},
			{
				"name": "Log Upload Start",
				"type": "Lookup",
				"dependsOn": [
					{
						"activity": "Get File List",
						"dependencyConditions": [
							"Succeeded"
						]
					}
				],
				"policy": {
					"timeout": "7.00:00:00",
					"retry": 0,
					"retryIntervalInSeconds": 30,
					"secureOutput": false,
					"secureInput": false
				},
				"userProperties": [],
				"typeProperties": {
					"source": {
						"type": "AzureSqlSource",
						"sqlReaderStoredProcedureName": "[dbo].[LogUploadStart]",
						"storedProcedureParameters": {
							"FileCount": {
								"type": "Int32",
								"value": {
									"value": "@activity('Get File List').output.count",
									"type": "Expression"
								}
							},
							"TriggerId": {
								"type": "String",
								"value": {
									"value": "@pipeline().TriggerId",
									"type": "Expression"
								}
							}
						},
						"queryTimeout": "02:00:00",
						"partitionOption": "None"
					},
					"dataset": {
						"referenceName": "GetSetMetadata",
						"type": "DatasetReference"
					}
				}
			},
			{
				"name": "Log Upload Finish",
				"type": "SqlServerStoredProcedure",
				"dependsOn": [
					{
						"activity": "UploadFiles",
						"dependencyConditions": [
							"Succeeded"
						]
					}
				],
				"policy": {
					"timeout": "7.00:00:00",
					"retry": 0,
					"retryIntervalInSeconds": 30,
					"secureOutput": false,
					"secureInput": false
				},
				"userProperties": [],
				"typeProperties": {
					"storedProcedureName": "[dbo].[LogUploadFinish]",
					"storedProcedureParameters": {
						"LogId": {
							"value": {
								"value": "@activity('Log Upload Start').output.firstRow.LogId",
								"type": "Expression"
							},
							"type": "Int32"
						}
					}
				},
				"linkedServiceName": {
					"referenceName": "trainingdb01",
					"type": "LinkedServiceReference"
				}
			}
		],
		"variables": {
			"DemoVariable": {
				"type": "String"
			}
		},
		"folder": {
			"name": "Demo Pipelines/Data Uploads"
		},
		"annotations": [],
		"lastPublishTime": "2020-09-02T14:13:14Z"
	},
	"type": "Microsoft.DataFactory/factories/pipelines"
}