我正在对IIS应用程序进行一些性能/可伸缩性测试,这些测试偶尔似乎会减慢到生产中的爬行速度。我能够使用NUnit一致地重现慢度。
CPU和内存在测试期间或生产中发生缓慢时不会出现峰值。我强烈怀疑应用程序正在遭受线程饥饿,因为它似乎不是导致瓶颈的CPU,内存,I / O或数据库访问。我确实看到出现的迹象是线程饥饿;例如,NLog的异步日志文件写入往往具有长时间的静默,随后是具有较旧时间戳的活动突发(即,较低优先级的线程正在等待线程释放以便写入)。
我可以采取哪些步骤来明确确定应用程序确实是线程缺乏的,并且(假设是这种情况)查明导致问题的系统的确切区域?
修改
我忽略了几乎所有代码都是同步的(它是遗留系统)。
答案 0 :(得分:7)
根据Sinatr的评论,我在ThreadPool.SetMinThreads和TaskCreationOptions.LongRunning上做了一些阅读,包括When to use TaskCreationOptions.LongRunning?的答案
将MinThreads设置为更高的默认值会对我的情况产生巨大影响。我创建了一个简单的后台进程,以查看ThreadPool中的可用线程在测试运行过程中是否发生了显着变化,并且超过了MinThreads值(它是)。
这是我用来诊断的一些代码。这不适用于生产用途,此处显示的线程使用情况报告只会在最初增加时才会引起注意。另请注意,Timer在经过时需要一个线程,因此还需要等待可用的线程。
静态变种:
int completionPortThreads;
ThreadPool.GetMaxThreads(out _maxThreads, out completionPortThreads);
ThreadPool.GetMinThreads(out _minThreads, out completionPortThreads);
_timer = new Timer
{
AutoReset = true,
Interval = 500,
};
_timer.Elapsed += TimerElasped;
_timer.Start();
在启动时运行:
private static void TimerElasped(object sender, ElapsedEventArgs e)
{
int minWorkerThreads;
int availWorkerThreads;
int completionPortThreads;
ThreadPool.GetMinThreads(out minWorkerThreads, out completionPortThreads);
ThreadPool.GetAvailableThreads(out availWorkerThreads, out completionPortThreads);
var activeThreads = _maxThreads - availWorkerThreads;
if (availWorkerThreads != _lastAvailableThreads)
{
_lastAvailableThreads = availWorkerThreads;
if (activeThreads > _lastActiveThreads)
{
_lastActiveThreads = activeThreads;
Logger.Log($"+++++ Active Threads is now: {activeThreads}");
if (activeThreads > _minThreads)
{
var diff = activeThreads - _minThreads;
Logger.Log($"+++++ Active threads is now {activeThreads}, which is {diff} more than minThread value of {_minThreads}. This may be causing delays.");
}
}
}
}
经过的方法:
<android.support.design.widget.TextInputLayout
android:id="@+id/enter_template_name_edit_text_input_layout"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textColorHint="@color/warm_grey">
<android.support.design.widget.TextInputEditText
android:id="@+id/template_name_text_input_edit_text"
style="@style/EditTextBaseStyle"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:gravity="start"
android:hint="@string/add_nitrogen_template_name_label"
android:imeOptions="actionNext"
android:inputType="textCapWords|textNoSuggestions"
android:maxLines="1"
android:paddingBottom="@dimen/md_content_padding_bottom"
android:textAlignment="textStart"
android:textColorHint="@color/warm_grey"
android:textSize="32sp" />
答案 1 :(得分:2)
我基于以上
提出了这个问题using System;
using System.Threading;
using System.Timers;
using log4net;
using Timer = System.Timers.Timer;
namespace somewhere
{
public class ThreadStatsLogger : IDisposable
{
private const int DEPLETION_WARN_LEVEL = 10;
private const int HISTERESIS_LEVEL = 10;
private const double SAMPLE_RATE_MILLISECONDS = 500;
private bool _workerThreadWarned = false;
private bool _ioThreadWarned = false;
private bool _minWorkerThreadLevelWarned = false;
private bool _minIoThreadLevelWarned = false;
private readonly int _maxWorkerThreadLevel;
private readonly int _maxIoThreadLevel;
private readonly int _minWorkerThreadLevel;
private readonly int _minWorkerThreadLevelRecovery;
private readonly int _minIoThreadLevel;
private readonly int _minIoThreadLevelRecovery;
private Timer _timer;
private static readonly ILog _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
public ThreadStatsLogger()
{
_timer = new Timer
{
AutoReset = true,
Interval = SAMPLE_RATE_MILLISECONDS,
};
_timer.Elapsed += TimerElasped;
_timer.Start();
ThreadPool.GetMinThreads(out _minWorkerThreadLevel, out _minIoThreadLevel);
ThreadPool.GetMaxThreads(out _maxWorkerThreadLevel, out _maxIoThreadLevel);
ThreadPool.GetAvailableThreads(out int workerAvailable, out int ioAvailable);
_logger.InfoFormat("Thread statistics at startup: minimum worker:{0} io:{1}", _minWorkerThreadLevel, _minIoThreadLevel );
_logger.InfoFormat("Thread statistics at startup: maximum worker:{0} io:{1}", _maxWorkerThreadLevel, _maxIoThreadLevel);
_logger.InfoFormat("Thread statistics at startup: available worker:{0} io:{1}", workerAvailable, ioAvailable);
_minWorkerThreadLevelRecovery = (_minWorkerThreadLevel * 3) / 4;
_minIoThreadLevelRecovery = (_minIoThreadLevel * 3) / 4;
if (_minWorkerThreadLevelRecovery == _minWorkerThreadLevel) _minWorkerThreadLevelRecovery = _minWorkerThreadLevel - 1;
if (_minIoThreadLevelRecovery == _minIoThreadLevel) _minIoThreadLevelRecovery = _minIoThreadLevel - 1;
}
private void TimerElasped(object sender, ElapsedEventArgs e)
{
ThreadPool.GetAvailableThreads(out int availableWorkerThreads, out int availableIoThreads);
var activeWorkerThreads = _maxWorkerThreadLevel - availableWorkerThreads;
var activeIoThreads = _maxIoThreadLevel - availableIoThreads;
_logger.InfoFormat("Thread statistics: active worker:{0} io:{1}", activeWorkerThreads, activeIoThreads);
if (activeWorkerThreads > _minWorkerThreadLevel && !_minWorkerThreadLevelWarned)
{
_logger.InfoFormat("Thread statistics WARN active worker threads above minimum {0}:{1}", activeWorkerThreads, _minWorkerThreadLevel);
_minWorkerThreadLevelWarned = !_minWorkerThreadLevelWarned;
}
if (activeWorkerThreads < _minWorkerThreadLevelRecovery && _minWorkerThreadLevelWarned)
{
_logger.InfoFormat("Thread statistics RECOVERY active worker threads below minimum {0}:{1}", activeWorkerThreads, _minWorkerThreadLevel);
_minWorkerThreadLevelWarned = !_minWorkerThreadLevelWarned;
}
if (activeIoThreads > _minIoThreadLevel && !_minIoThreadLevelWarned)
{
_logger.InfoFormat("Thread statistics WARN active io threads above minimum {0}:{1}", activeIoThreads, _minIoThreadLevel);
_minIoThreadLevelWarned = !_minIoThreadLevelWarned;
}
if (activeIoThreads < _minIoThreadLevelRecovery && _minIoThreadLevelWarned)
{
_logger.InfoFormat("Thread statistics RECOVERY active io threads below minimum {0}:{1}", activeIoThreads, _minIoThreadLevel);
_minIoThreadLevelWarned = !_minIoThreadLevelWarned;
}
if (availableWorkerThreads < DEPLETION_WARN_LEVEL && !_workerThreadWarned)
{
_logger.InfoFormat("Thread statistics WARN available worker threads below warning level {0}:{1}", availableWorkerThreads, DEPLETION_WARN_LEVEL);
_workerThreadWarned = !_workerThreadWarned;
}
if (availableWorkerThreads > (DEPLETION_WARN_LEVEL + HISTERESIS_LEVEL) && _workerThreadWarned)
{
_logger.InfoFormat("Thread statistics RECOVERY available worker thread recovery {0}:{1}", availableWorkerThreads, DEPLETION_WARN_LEVEL);
_workerThreadWarned = !_workerThreadWarned;
}
if (availableIoThreads < DEPLETION_WARN_LEVEL && !_ioThreadWarned)
{
_logger.InfoFormat("Thread statistics WARN available io threads below warning level {0}:{1}", availableIoThreads, DEPLETION_WARN_LEVEL);
_ioThreadWarned = !_ioThreadWarned;
}
if (availableIoThreads > (DEPLETION_WARN_LEVEL + HISTERESIS_LEVEL) && _ioThreadWarned)
{
_logger.InfoFormat("Thread statistics RECOVERY available io thread recovery {0}:{1}", availableIoThreads, DEPLETION_WARN_LEVEL);
_ioThreadWarned = !_ioThreadWarned;
}
}
public void Dispose()
{
if (_timer == null) return;
_timer.Close();
_timer.Dispose();
_timer = null;
}
}
}