Question

我一直在尝试按照本文档中的步骤https://docs.microsoft.com/en-us/azure/data-factory/v1/data-factory-use-custom-activities 创建自定义活动，但它会在输出数据集中抛出运行超时错误：

管道：

{
"name": "ADFTutorialPipelineCustom",
"properties": {
    "description": "Use custom activity",
    "activities": [
        {
            "type": "DotNetActivity",
            "typeProperties": {
                "assemblyName": "MyDotNetActivity.dll",
                "entryPoint": "MyDotNetActivityNS.MyDotNetActivity",
                "packageLinkedService": "Destination-BlobStorage",
                "packageFile": "blobcontainer/MyDotNetActivity.zip",
                "extendedProperties": {
                    "SliceStart": "$$Text.Format('{0:yyyyMMddHH-mm}', Time.AddMinutes(SliceStart, 0))"
                }
            },
            "outputs": [
                {
                    "name": "OutputDataset"
                }
            ],
            "policy": {
                "timeout": "00:30:00",
                "concurrency": 1,
                "retry": 3
            },
            "scheduler": {
                "frequency": "Minute",
                "interval": 30
            },
            "name": "MyDotNetActivity",
            "linkedServiceName": "AzureBatchAccount"
        }
    ],
    "start": "2018-01-09T06:00:00Z",
    "end": "2018-01-11T07:00:00Z",
    "isPaused": false,
    "hubName": "test2017dec11_hub",
    "pipelineMode": "Scheduled"
}

输出数据集：

{
"name": "OutputDataset",
"properties": {
    "published": false,
    "type": "AzureBlob",
    "linkedServiceName": "Destination-BlobStorage",
    "typeProperties": {
        "fileName": "{slice}.txt",
        "folderPath": "blobcontainer",
        "partitionedBy": [
            {
                "name": "slice",
                "value": {
                    "type": "DateTime",
                    "date": "SliceStart",
                    "format": "yyyy-MM-dd-HH"
                }
            }
        ]
    },
    "availability": {
        "frequency": "Minute",
        "interval": 30
    }
}

活动代码：

    public class MyDotNetActivity : IDotNetActivity
{
    public IDictionary<string, string> Execute(IEnumerable<LinkedService> linkedServices, IEnumerable<Dataset> datasets, Activity activity, IActivityLogger logger)
    {
        logger.Write("Activity start");

        AzureStorageLinkedService outputLinkedService;

        Dataset outputDataset = datasets.Single(dataset => dataset.Name == activity.Outputs.Single().Name);

        outputLinkedService = linkedServices.First(
            linkedService =>
            linkedService.Name ==
            outputDataset.Properties.LinkedServiceName).Properties.TypeProperties
            as AzureStorageLinkedService;

        string connectionString = outputLinkedService.ConnectionString;

        string folderPath = GetFolderPath(outputDataset);

        UploadFileToBlob(connectionString, folderPath, GetFileName(outputDataset), logger);

        return new Dictionary<string, string>();
    }

    private static string GetFolderPath(Dataset dataArtifact)
    {
        if (dataArtifact == null || dataArtifact.Properties == null)
        {
            return null;
        }
        AzureBlobDataset blobDataset = dataArtifact.Properties.TypeProperties as AzureBlobDataset;
        if (blobDataset == null)
        {
            return null;
        }
        return blobDataset.FolderPath;
    }

    public void UploadFileToBlob(string blobConnectionString, string blobFolderPath, string fileName, IActivityLogger logger)
    {
        logger.Write("connecting to the blob..");
        var outputStorageAccount = CloudStorageAccount.Parse(blobConnectionString);
        string output = string.Empty;
        output += "test blob storage";

        var outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, blobFolderPath + "/" + fileName);
        var outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials);
        logger.Write("uploading to the blob URI: {0}", outputBlobUri.ToString());
        outputBlob.UploadText(output);
        logger.Write("upload succeeded");
    }
}

请告知如何解决这个问题，谢谢。

Answer 1

根据您显示的内容，您的活动政策超时设置为30分钟。要更改此设置，请在管道中编辑现有活动策略超时，并将其设置为更大的值。即

with demo (id, name) as
    ( select 1, 'Matt' from dual union all
      select 2, 'Ryan' from dual union all
      select 3, 'Joseph' from dual union all
      select 4, 'Matt1' from dual union all
      select 5, '5Joseph' from dual union all
      select 6, 'David' from dual union all
      select 7, 'Matt_43' from dual )
select id, name
     , soundex(name)
     , count(*) over(partition by soundex(name)) as count_similar
     , utl_match.jaro_winkler_similarity(name, 'Matt') as similarity_to_matt
from   demo
order by similarity_to_matt desc;

     ID NAME      SOUNDEX(NAME) COUNT_SIMILAR SIMILARITY_TO_MATT
------- --------- ------------- ------------- ------------------
      1 Matt      M300                      3                100
      4 Matt1     M300                      3                 96
      7 Matt_43   M300                      3                 91
      2 Ryan      R500                      1                 50
      6 David     D130                      1                 48
      5 5Joseph   J210                      2                  0
      3 Joseph    J210                      2                  0

要执行此操作，请转到数据工厂，然后单击“作业和部署”操作：http://prntscr.com/hxjr9n，然后单击“管道http://prntscr.com/hxjrie”并在我编写时编辑相关项的JSON。

Azure数据工厂管道中的Azure自定义活动超时

1 个答案: