Question

我试图使用多线程来大大减少我的MySQL工作时间。我有几台具有700多个DB的服务器，每个服务器上需要运行〜22个查询。

我尝试使用MySQLdb，asyncio和ThreadPoolExecutor，但一直收到分段错误。老实说，我既不是确定这是最好的方法，也不是我确定编码正确。

#!/usr/bin/env python3

import asyncio
from concurrent.futures import ThreadPoolExecutor
from logging.handlers import SysLogHandler
import sys
import os
import configparser
import datetime
import logging
from logging.handlers import SysLogHandler
import MySQLdb

def setup():

    #Some Setup stuff here

    event_loop = asyncio.get_event_loop()

    try:
        event_loop.run_until_complete(Main(dbServers, mySQLuser, mySQLpass, db_ignore_list, trunc_date.strftime("%Y-%m-%d %H:%M:%S"), logger))
    finally:
        event_loop.close()

async def Main(dbServers, mySQLuser, mySQLpass, db_ignore_list, trunc_date, logger):
    try:
        db = MySQLdb.connect(host=dbServers, user=mySQLuser, passwd=mySQLpass)
    except:
        logger.critical('Error connecting to the Database server {}'.format(dbServers))
        quit()

    cur = db.cursor()
    cur.execute('SHOW DATABASES')
    databases = [item[0] for item in cur.fetchall()]
    db_list = list(set(databases) - set(db_ignore_list))

    # Generator Object for .map
    args = ((db,s,trunc_date) for s in db_list)

    with ThreadPoolExecutor(max_workers=5) as executor:
        executor.map(lambda p: archive_table(*p), args)
    db.close()

def archive_table(db, schema, trunc_date):
    cur = db.cursor()
    db.select_db(schema)
    cur.execute('SOME SQL') #x22
    cur.execute('SOME SQL') #x22
    cur.execute('SOME SQL') #x22
    cur.execute('SOME SQL') #x22
    db.commit()


if __name__ == '__main__':
    setup()```

Segmentation fault (core dumped)

Python：如何在多个数据库中对许多查询进行多线程处理

0 个答案: