具有参数化数据集和参数化链接服务的Azure数据流

时间:2019-08-26 16:37:21

标签: azure-data-factory

我有一个带有2个Azure SQL链接服务的ADF模板,这些服务具有用于数据库的参数和2个相应的数据集,它们具有用于表的附加参数。我在复制活动中使用了这些链接服务,从而成功地将一些数据从一个sql数据库移动到了另一个sql数据库。 我现在正在尝试使用非常简单的数据流来实现相同的任务,该数据流使用与源和表和数据库参数同步的参数,并使用与成功复制活动相同的数据集和链接服务,但是我一直收到错误请求错误我无法修复。即使我对参数值进行硬编码,它也不起作用。 我正在从powershell部署ADF。

{
        "name": "[concat(parameters('factoryName'), '/AzureSqlDatabaseTenant')]",
        "type": "Microsoft.DataFactory/factories/linkedServices",
        "apiVersion": "2018-06-01",
        "properties": {
            "parameters": {
                "DBName": {
                    "type": "String"
                }
            },
            "annotations": [],
            "type": "AzureSqlDatabase",
            "typeProperties": {
                "connectionString": "[parameters('AzureSqlDatabaseTenant_connectionString')]"
            }
        },
        "dependsOn": []
    }
{
            "name": "[concat(parameters('factoryName'), '/AzureSqlDataWarehouse')]",
            "type": "Microsoft.DataFactory/factories/linkedServices",
            "apiVersion": "2018-06-01",
            "properties": {
                "parameters": {
                    "DBName": {
                        "type": "String"
                    }
                },
                "annotations": [],
                "type": "AzureSqlDatabase",
                "typeProperties": {
                    "connectionString": "[parameters('AzureSqlDataWarehouse_connectionString')]"
                }
            },
            "dependsOn": []
        }
{
            "name": "[concat(parameters('factoryName'), '/dataflow1')]",
            "type": "Microsoft.DataFactory/factories/dataflows",
            "apiVersion": "2018-06-01",
            "properties": {
                "type": "MappingDataFlow",
                "typeProperties": {
                    "sources": [
                        {
                            "dataset": {
                                "referenceName": "AzureSqlDatabaseTenant",
                                "type": "DatasetReference"
                            },
                            "name": "source1",
                            "typeProperties": {}
                        }
                    ],
                    "sinks": [
                        {
                            "dataset": {
                                "referenceName": "AzureSqlDataWarehouse",
                                "type": "DatasetReference"
                            },
                            "name": "sink1"
                        }
                    ],
                    "transformations": [
                        {
                            "name": "Select1"
                        }
                    ],
                    "script": "\nparameters{\n\tTenantDBName as string ('contosoconcerthall'),\n\tWarehouseDBName as string ('DWTenant1')\n}\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> source1\nsource1 select() ~> Select1\nSelect1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> sink1"
                }
            },
            "dependsOn": [
                "[concat(variables('factoryId'), '/datasets/AzureSqlDataWarehouse')]",
                "[concat(variables('factoryId'), '/datasets/AzureSqlDatabaseTenant')]"
            ]
        }
{
            "name": "[concat(parameters('factoryName'), '/pipeline1')]",
            "type": "Microsoft.DataFactory/factories/pipelines",
            "apiVersion": "2018-06-01",
            "properties": {
                "activities": [
                    {
                        "name": "dataflow1",
                        "type": "ExecuteDataFlow",
                        "dependsOn": [],
                        "policy": {
                            "timeout": "7.00:00:00",
                            "retry": 0,
                            "retryIntervalInSeconds": 30,
                            "secureOutput": false,
                            "secureInput": false
                        },
                        "userProperties": [],
                        "typeProperties": {
                            "dataflow": {
                                "referenceName": "dataflow1",
                                "type": "DataFlowReference",
                                "parameters": {
                                    "TenantDBName": "'contosoconcerthall'",
                                    "WarehouseDBName": "'DWTenant1'"
                                },
                                "datasetParameters": {
                                    "source1": {
                                        "DBName": "contosoconcerthall",
                                        "TableName": "Venue"
                                    },
                                    "sink1": {
                                        "DBName": "DWTenant1",
                                        "TableName": "dim_Venues"
                                    }
                                }
                            },
                            "staging": {}
                        }
                    }
                ],
                "parameters": {
                    "TenantDBName": {
                        "type": "string"
                    },
                    "WarehouseDBName": {
                        "type": "string"
                    }
                },
                "annotations": []
            },
            "dependsOn": [
                "[concat(variables('factoryId'), '/dataflows/dataflow1')]"
            ]
        }
# Creating connection strings SQL tenant and warehouse databases
        $dbconnection = "Server=tcp:" + $fullyQualifiedTenantServerName + ",1433;Database=@{linkedService().DBName};User ID=" + $tenantSqlServerConfig.AdminSqlLogin + "@" + $tenantSqlServerConfig.ServerName + ";Password=" + $tenantSqlServerConfig.AdminSqlPassword + ";Trusted_Connection=False;Encrypt=True;Connection Timeout=90"
        $dwconnection = "Server=tcp:" + $fullyQualifiedWarehouseServerName + ",1433;Database=@{linkedService().DBName};User ID=" + $sqlServerConfig.AdminSqlLogin + "@" + $sqlServerConfig.ServerName + ";Password=" + $sqlServerConfig.AdminSqlPassword + ";Trusted_Connection=False;Encrypt=True;Connection Timeout=90"

        # Converting to secure string
        $secureStringdbconnection = ConvertTo-SecureString $dbconnection -AsPlainText -Force
        $secureStringdwconnection = ConvertTo-SecureString $dwconnection -AsPlainText -Force

        $deployment = New-AzResourceGroupDeployment `
                -TemplateFile ($PSScriptRoot + "\..\Template\" + $dfConfig.DataFactoryDeploymentTemplate) `
                -ResourceGroupName $dfConfig.ResourceGroupName `
                -FactoryName $dfConfig.DataFactoryName `
                -TenantAzureSqlDatabase_connectionString $secureStringdbconnection `
                -TenantAzureSqlWarehouse_connectionString $secureStringdwconnection `
                -ErrorAction Stop 

0 个答案:

没有答案