我正在实时执行多个PostgreSQL更新：每秒约50次写入。
这是我在数据库中编写的代码，我在写入数据库后每次关闭连接：
def update_database(sqlquery):
"""
:rtype : object
:param sqlquery:
:return:
"""
try:
db = DatabaseSession.DatabaseSession(settings.SQLALCHEMY_DATABASE_URI)
db.execute(sqlquery)
except psycopg2.ProgrammingError, exception:
print exception
except Exception, exception:
print exception
数据库会话：
class DatabaseSession(object):
"""
"""
def __init__(self, db_url):
"""
:param db_url:
:return:
"""
self.db_url = db_url
self.session = None
self.engine = None
def get_session(self):
"""
:return:
"""
try:
self.engine = create_engine(self.db_url,
pool_recycle=settings.SQLALCHEMY_POOL_RECYCLE,
pool_size=settings.SQLALCHEMY_POOL_SIZE,
max_overflow=settings.SQLALCHEMY_MAX_OVERFLOW)
Session = sessionmaker()
Session.configure(bind=self.engine)
self.session = Session()
self.session._model_changes = {}
return self.session
except Exception, e:
log.error('DatabaseSession() get_session(): ' + str(e.message))
def execute(self, sqlcommand):
"""
:param sqlcommand:
:return:
"""
try:
session_active = False
connection = None
# Check if session
if self.session:
session_active = True
else:
if self.get_session():
session_active = True
else:
session_active = False
if session_active:
connection = self.engine.connect()
result = connection.execute(sqlcommand)
return result
else:
log.error('DatabaseSession() Unable to start session. Session is not active')
except psycopg2.OperationalError as exception:
log.exception('DatabaseSession() OperationalError(): {}'.format(exception))
except Exception as exception:
log.exception('DatabaseSession() Execute(): '.format(exception))
finally:
if connection:
connection.close()
设定：
SQLALCHEMY_DATABASE_URI = "postgresql://" + dbusername + ":" + dbpassword + "@" + dbhost + "/" + dbname
SQLALCHEMY_ECHO = False
SQLALCHEMY_POOL_SIZE = 1000
SQLALCHEMY_POOL_TIMEOUT = 5
SQLALCHEMY_MAX_OVERFLOW = 100
SQLALCHEMY_POOL_RECYCLE = 60
SQLALCHEMY_TRACK_MODIFICATIONS = True
DATABASE_CONNECT_OPTIONS = None
SQLALCHEMY_COMMIT_ON_TEARDOWN = True
几分钟后我得到了这个：
[2016-07-21 03:23:44,581: ERROR/Worker-7] DatabaseSession() Execute():
Traceback (most recent call last):
File "/usr/src/telephonist/api/version1_0/database/DatabaseSession.py", line 65, in execute
connection = self.engine.connect()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2018, in connect
return self._connection_cls(self, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 72, in __init__
if connection is not None else engine.raw_connection()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2104, in raw_connection
self.pool.unique_connection, _connection)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2078, in _wrap_pool_connect
e, dialect, self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1405, in _handle_dbapi_exception_noconnection
exc_info
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2074, in _wrap_pool_connect
return fn()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 318, in unique_connection
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 713, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 480, in checkout
rec = pool._do_get()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 1060, in _do_get
self._dec_overflow()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 1057, in _do_get
return self._create_connection()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 323, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 449, in __init__
self.connection = self.__connect()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 607, in __connect
connection = self.__pool._invoke_creator(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 97, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 385, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib/python2.7/dist-packages/psycopg2/__init__.py", line 164, in connect
conn = _connect(dsn, connection_factory=connection_factory, async=async)
OperationalError: (psycopg2.OperationalError) could not fork new process for connection: Cannot allocate memory
could not fork new process for connection: Cannot allocate memory
could not fork new process for connection: Cannot allocate memory
我调整了数据库设置：
max_connections = 1024 # (change requires restart)
shared_buffers = 512MB # min 128kB
# (change requires restart)
temp_buffers = 8MB # min 800kB
work_mem = 1MB # min 64kB
相关帖子：
Heroku Rails could not fork new process for connection: Cannot allocate memory
Django OperationalError: could not fork new process for connection