Question

我有这段代码：

from twisted.internet import reactor
from twisted.internet import defer, task
from pysnmp.entity import engine, config
from pysnmp.carrier.twisted import dispatch
from pysnmp.carrier.twisted.dgram import udp
from pysnmp.entity.rfc3413.twisted import cmdgen


import __webimport__
import tools.config
from tools.database import makedsn
import psycopg2


def cmp_varBinds(varBind, varName):
    if varName[0] in str(varBind[0]):
        return True


def cbFun(cbCtx, ip, varNames):
    (errorIndication, errorStatus, errorIndex, varBinds) = cbCtx
    if varBinds and any(map(cmp_varBinds, varBinds[0], varNames)):
        print ip, [str(x[1]) for x in varBinds[0]]
        df = defer.Deferred()
        df.addCallback(cbFun, ip=ip, varNames=varNames)
        return df  # This also indicates that we wish to continue walkin


def parallel(iterable, count, callable, *args, **named):
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])





def fetch(host):
    id, ip, community, hc = host
    snmpEngine = engine.SnmpEngine()
    snmpEngine.registerTransportDispatcher(dispatch.TwistedDispatcher())
    config.addV1System(snmpEngine, 'test-agent', community)
    config.addTargetParams(snmpEngine, 'myParams', 'test-agent', 'noAuthNoPriv', 1)

    config.addTargetAddr(
                     snmpEngine, 'myRouter', config.snmpUDPDomain,
                     (ip, 161), 'myParams', timeout=1
    )

    # Transport
    config.addSocketTransport(
                          snmpEngine,
                          udp.domainName,
                          udp.UdpTwistedTransport().openClientMode()
    )

    getCmdGen = cmdgen.NextCommandGenerator()
    varNames = [('1.3.6.1.2.1.2.2.1.11', None),
            ('1.3.6.1.2.1.2.2.1.12', None),
            ('1.3.6.1.2.1.2.2.1.13', None),
            ('1.3.6.1.2.1.2.2.1.14', None)]
    df = getCmdGen.sendReq(snmpEngine, 'myRouter', varNames)
    df.addCallback(cbFun, ip=ip, varNames=varNames)
    return df


dsn = makedsn(tools.config.main_db)
connection = psycopg2.connect(dsn)
cursor = connection.cursor()
cursor.execute("""SELECT e.id, e.ip, e.snmpcomm, e.hccnt
           FROM snmp_ports sp, equipment e
           WHERE e.snmp = 'Y' and sp.equipment = e.id
           GROUP BY e.id,e.ip,e.snmpcomm,e.hccnt
           ORDER BY e.id""")
hosts = cursor.fetchall()


finished = parallel(hosts, len(hosts), fetch)
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
reactor.run()

我从数据库中获取4000个主机并询问每个主机。如果我在SQL查询中设置限制1000它工作正常。但是当主机超过1000时我得到一个错误：

Unhandled Error
Traceback (most recent call last):
  File "crawler.py", line 98, in <module>

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1192, in run

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1201, in mainLoop

--- <exception caught here> ---
  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/twisted/internet/base.py", line 824, in runUntilCurrent

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/carrier/base.py", line 52, in _cbFun

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/entity/engine.py", line 64, in __receiveMessageCbFun

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/proto/rfc3412.py", line 274, in receiveMessage

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/smi/builder.py", line 299, in importSymbols

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/smi/builder.py", line 270, in loadModules

pysnmp.smi.error.SmiError: MIB file "__SNMPv2-MIB.py[co]" not found in search path

然后脚本停止。为什么会这样？

Answer 1

你正在使用什么pysnmp？确保您使用的是最新的pysnmp版本。

作为旁注 - 您的方法效率极低，因为您似乎在每次GET操作时重新初始化[重] SnmpEngine。更好的方法是为每个进程/线程保留一个持久的SnmpEngine实例。

Answer 2

如果只有在增加并发级别时才会发生这种情况，那么可能的解释是，您已经遇到了平台在任何时候都可以拥有的打开文件数量的限制。每个打开的套接字都会计入此限制，打开“常规”文件（来自文件系统）也是如此。

如果你用完了所有允许的文件，那么Python无法读取磁盘上的模块的源代码，因为该平台不会让它打开它们。

在这种情况下发生的事情并不是很明显，因为（如果是）pysnmp正在处理真正的异常并重新引发一个隐藏细节的新异常。

如果这是问题，那么您可以通过提高打开文件限制来解决它。大多数情况下，您可以通过运行：

来完成此操作

$ ulimit -Sn 2048

有关控制该限制的详细信息，请详细了解您的shell中的ulimit（help ulimit，the internet）。

pysnmp扭曲客户端中有超过1000个主机的未处理错误

2 个答案: