从PANDAS导入MS SQL Server CSV速度很慢。还有其他选择吗?

时间:2019-07-18 17:59:10

标签: python sql-server csv pyodbc

我的代码通过PYODBC将CSV加载到MS SQL Server中。但是,清洁后的CSV为300MB + 300万。行。这变得非常缓慢。这是我的代码。您知道如何加快流程吗?谢谢。

def impserv1251(self):
    try:  
        import pyodbc
        import os
        import sys
        import struct
        import fileinput
        import csv
        import pandas as pd
        serv = 'xxxxxxxxx\xxxxxxxx'
        datab = 'xxxxxxxxxxxxxxxx' 
        cnxn = pyodbc.connect('Trusted_Connection=yes', driver = '{ODBC Driver 13 for SQL Server}',server = serv, database = datab)
        print(cnxn)
        cwd = 'xxxxxxxxxxxxxxxxxxxxxxxxx'
        directory = (cwd + '\\FINAL\\CSV')
        for file in os.listdir(directory):
            if file.endswith( "1251_FINAL.csv"):
                data = pd.read_csv(directory + '\\' + "1251_FINAL.csv", sep=",", usecols=['SYS', 'MANDT', 'AGR_NAME', 'OBJECT', 'AUTH', 'FIELD', 'LOW', 'HIGH', 'DELETED'], encoding='latin-1', dtype={'SYS':str, 'MANDT':str, 'AGR_NAME':str, 'OBJECT':str, 'AUTH':str, 'FIELD':str, 'LOW':str, 'HIGH':str, 'DELETED':str})
                data = data.fillna(value='')       
        cursor = cnxn.cursor()
        for pos, row in data.iterrows():
                cursor.execute("INSERT INTO dbo.AGR_1251_ALL([SYS], [MANDT], [AGR_NAME], [OBJECT], [AUTH], [FIELD], [LOW], [HIGH], [DELETED]) values(?,?,?,?,?,?,?,?,?)", row['SYS'], row['MANDT'], row['AGR_NAME'], row['OBJECT'], row['AUTH'], row['FIELD'], row['LOW'], row['HIGH'], row['DELETED'])
        cnxn.commit()
        cursor.close()
        cnxn.close()

0 个答案:

没有答案