随机Azure功能应用程序失败：超出主机阈值[连接]

Question

我有以下功能应用

[FunctionName("SendEmail")]
public static async Task Run([ServiceBusTrigger("%EmailSendMessageQueueName%", AccessRights.Listen, Connection = AzureFunctions.Connection)] EmailMessageDetails messageToSend,
    [ServiceBus("%EmailUpdateQueueName%", AccessRights.Send, Connection = AzureFunctions.Connection)]IAsyncCollector<EmailMessageUpdate> messageResponse,
    //TraceWriter log,
    ILogger log,
    CancellationToken token)
{
    log.LogInformation($"C# ServiceBus queue trigger function processed message: {messageToSend}");

    /* Validate input and initialise Mandrill */
    try
    {
        if (!ValidateMessage(messageToSend, log))   // TODO: finish validation
        {
            log.LogError("Invalid or Unknown Message Content");
            throw new Exception("Invalid message content.");
        }
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to Validate Message data: {ex.Message} => {ex.ReportAllProperties()}");
        throw;
    }

    DateTime utcTimeToSend;
    try
    {
        var envTag = GetEnvVariable("Environment");
        messageToSend.Tags.Add(envTag);

        utcTimeToSend = messageToSend.UtcTimeToSend.GetNextUtcSendDateTime();
        DateTime utcExpiryDate = messageToSend.UtcTimeToSend.GetUtcExpiryDate();
        DateTime now = DateTime.UtcNow;
        if (now > utcExpiryDate)
        {
            log.LogError($"Stopping sending message because it is expired: {utcExpiryDate}");
            throw new Exception($"Stopping sending message because it is expired: {utcExpiryDate}");
        }
        if (utcTimeToSend > now)
        {
            log.LogError($"Stopping sending message because it is not allowed to be send due to time constraints: next send time: {utcTimeToSend}");
            throw new Exception($"Stopping sending message because it is not allowed to be send due to time constraints: next send time: {utcTimeToSend}");
        }
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to Parse and/or Validate Message Time To Send: {ex.Message} => {ex.ReportAllProperties()}");
        throw;
    }

    /* Submit message to Mandrill */
    string errorMessage = null;
    IList<MandrillSendMessageResponse> mandrillResult = null;
    DateTime timeSubmitted = default(DateTime);
    DateTime timeUpdateRecieved = default(DateTime);

    try
    {
        var mandrillApi = new MandrillApi(GetEnvVariable("Mandrill:APIKey"));

        var mandrillMessage = new MandrillMessage
        {
            FromEmail = messageToSend.From,
            FromName = messageToSend.FromName,
            Subject = messageToSend.Subject,
            TrackClicks = messageToSend.Track,
            Tags = messageToSend.Tags,
            TrackOpens = messageToSend.Track,
        };

        mandrillMessage.AddTo(messageToSend.To, messageToSend.ToName);
        foreach (var passthrough in messageToSend.PassThroughVariables)
        {
            mandrillMessage.AddGlobalMergeVars(passthrough.Key, passthrough.Value);
        }

        timeSubmitted = DateTime.UtcNow;
        if (String.IsNullOrEmpty(messageToSend.TemplateId))
        {
            log.LogInformation($"No Message Template");
            mandrillMessage.Text = messageToSend.MessageBody;
            mandrillResult = await mandrillApi.Messages.SendAsync(mandrillMessage, async: true, sendAtUtc: utcTimeToSend);
        }
        else
        {
            log.LogInformation($"Using Message Template: {messageToSend.TemplateId}");
            var clock = new Stopwatch();
            clock.Start();
            mandrillResult = await mandrillApi.Messages.SendTemplateAsync(
                mandrillMessage,
                messageToSend.TemplateId,
                async: true,
                sendAtUtc: utcTimeToSend
            );
            clock.Stop();
            log.LogInformation($"Call to mandrill took {clock.Elapsed}");
        }
        timeUpdateRecieved = DateTime.UtcNow;
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to call Mandrill: {ex.Message} => {ex.ReportAllProperties()}");
        errorMessage = ex.Message;
    }

    try
    {
        MandrillSendMessageResponse theResult = null;
        SendMessageStatus status = SendMessageStatus.FailedToSendToProvider;

        if (mandrillResult == null || mandrillResult.Count < 1)
        {
            if (String.IsNullOrEmpty(errorMessage))
            {
                errorMessage = "Invalid Mandrill result.";
            }
        }
        else
        {
            theResult = mandrillResult[0];
            status = FacMandrillUtils.ConvertToSendMessageStatus(theResult.Status);
        }

        var response = new EmailMessageUpdate
        {
            SentEmailInfoId = messageToSend.SentEmailInfoId,
            ExternalProviderId = theResult?.Id ?? String.Empty,
            Track = messageToSend.Track,
            FacDateSentToProvider = timeSubmitted,
            FacDateUpdateRecieved = timeUpdateRecieved,
            FacErrorMessage = errorMessage,
            Status = status,
            StatusDetail = theResult?.RejectReason ?? "Error"
        };

        await messageResponse.AddAsync(response, token).ConfigureAwait(false);
    }
    catch (Exception ex)
    {
        log.LogError($"Failed to push message to the update ({AzureFunctions.EmailUpdateQueueName}) queue: {ex.Message} => {ex.ReportAllProperties()}");
        throw;
    }
}

当我排队100条消息时，一切运行正常。当我排队500条以上的邮件时，会发送499封邮件，但最后一封邮件永远不会被发送。我也开始出现以下错误。

  

操作被取消了。

我已经设置了Application Insights并进行了配置，并且我正在运行日志记录。我无法在本地重现，并根据Application Insights的以下端到端交易详情，我相信问题正在发生：

await messageResponse.AddAsync(response, token).ConfigureAwait(false);

应用洞察端到端交易

host.json

{
  "logger": {
    "categoryFilter": {
      "defaultLevel": "Information",
      "categoryLevels": {
        "Host": "Warning",
        "Function": "Information",
        "Host.Aggregator": "Information"
      }
    }
  },
  "applicationInsights": {
    "sampling": {
      "isEnabled": true,
      "maxTelemetryItemsPerSecond": 5
    }
  },
  "serviceBus": {
    "maxConcurrentCalls": 32
  }
}

同样相关的是来自Application Insights的错误。

[

还有其他人有这个或类似的问题吗？

Answer 1

如果您点击例外https://aka.ms/functions-thresholds中的链接，您将看到以下限制：

  

连接：出站连接数（限制为300）。有关处理连接限制的信息，请参阅管理连接。

你可能会击中那个。

在每个函数调用中，您将创建一个MandrillApi的新实例。您还没有提到您正在使用哪个库，但我怀疑它正在为MandrillApi的每个实例创建一个新连接。

我为每个实例检查了Mandrill Dot Net，是的，it's creating新的HttpClient：

_httpClient = new HttpClient
{
    BaseAddress = new Uri(BaseUrl)
};

Managing Connections建议：

  

在许多情况下，可以通过重用客户端实例而不是在每个函数中创建新实例来避免此连接限制。如果您使用单个静态客户端，则HttpClient，DocumentClient和Azure存储客户端等.NET客户端可以管理连接。如果使用每个函数调用重新实例化这些客户端，则代码很可能泄漏连接。

如果API客户端是线程安全的，请检查该库的文档，如果是，则在函数调用之间重用它。