Question

我有一个python脚本，它在每天开始时使用Task Scheduler运行，读取不断增长的日志文件（文本文件）并将数据插入Postgresql DB。每天生成新的日志文件。每个日志的大致大小为1GB。

平台：Windows 7，我不喜欢它，但这次我不喜欢）
RAM：32GB

我搜索了调整PostgreSQL以处理繁重的I / O，这是我修改的内容：

shared_buffers: 8GB 
work_mem: 100 MB 
maintenance_work_mem: 512 MB 
checkpoint_segments: 100 
checkpoint_timepot: 1hr 
synchronous_commit = off 
full_page_writes = off 
fsync = off

编写的脚本逐行读取日志文件并插入DB：

 import psycopg2 as psycopg
    try:
      connectStr = "dbname='postgis20' user='postgres' password='' host='localhost'"
      cx = psycopg.connect(connectStr)
      cu = cx.cursor()
      logging.info("connected to DB")
    except:
      logging.error("could not connect to the database")


import time
file = open('textfile.log', 'r')
while 1:
    where = file.tell()
    line = file.readline()
    if not line:
        time.sleep(1)
        file.seek(where)
    else:
        print line, # already has newline
        dodecode(line)

def dodecode(fields):
   global cx
   from time import strftime, gmtime
   from calendar import timegm
   import os
   msg = fields.split(',')
   part = eval(msg[2])
   msgnum = int(msg[3:6])
   print "message#:", msgnum
   print fields

   if (part==1):
     if msgnum==1:
       msg1 = msg_1.decode(bv)
       #print "message1 :",msg1
       Insert(msgnum,time,msg1)
     elif msgnum==2:
       msg2 = msg_2.decode(bv)
       #print "message2 :",msg2
       Insert(msgnum,time,msg2)
     elif msgnum==3:
     ....
     ....
     ....

def Insert(msgnum,time,msg):
 global cx

 try:    
         if msgnum in [1,2,3]:   
          if msg['type']==0:
            cu.execute("INSERT INTO table1 ( messageid, timestamp, userid, position, text ) SELECT "+str(msgnum)+", '"+time+"', "+str(msg['UserID'])+", ST_GeomFromText('POINT("+str(float(msg['longitude']), '"+text+"')+" "+str(float(msg['latitude']))+")']))+"  WHERE NOT EXISTS (SELECT * FROM table1 WHERE timestamp='"+time+"' AND text='"+text+"';")      
            cu.execute("INSERT INTO table2 ( field1,field2,field3, time_stamp, pos,) SELECT "+str(msg['UserID'])+","+str(int(msg['UserName']))+","+str(int(msg['UserIO']))+", '"+time+"', ST_GeomFromText('POINT("+str(float(msg['longitude']))+" "+str(float(msg['latitude']))+")')," WHERE NOT EXISTS (SELECT * FROM table2 WHERE field1="+str(msg['UserID'])+");")
            cu.execute("Update table2 SET field3='"+str(int(msg['UserIO']))+"',time_stamp='"+str(time)+"',pos=ST_GeomFromText('POINT("+str(float(msg['longitude']))+" "+str(float(msg['latitude']))+")'),"' WHERE field1='"+str(msg['UserID'])+"' AND time_stamp < '"+str(time)+"';")
          elif msg['type']==1:
            cu.execute("INSERT INTO table1 ( messageid, timestamp, userid, position, text ) SELECT "+str(msgnum)+", '"+time+"', "+str(msg['UserID'])+", ST_GeomFromText('POINT("+str(float(msg['longitude']), '"+text+"')+" "+str(float(msg['latitude']))+")']))+"  WHERE NOT EXISTS (SELECT * FROM table1 WHERE timestamp='"+time+"' AND text='"+text+"';")    
            cu.execute("INSERT INTO table2 ( field1,field2,field3, time_stamp, pos,) SELECT "+str(msg['UserID'])+","+str(int(msg['UserName']))+","+str(int(msg['UserIO']))+", '"+time+"', ST_GeomFromText('POINT("+str(float(msg['longitude']))+" "+str(float(msg['latitude']))+")')," WHERE NOT EXISTS (SELECT * FROM table2 WHERE field1="+str(msg['UserID'])+");")
            cu.execute("Update table2 SET field3='"+str(int(msg['UserIO']))+"',time_stamp='"+str(time)+"',pos=ST_GeomFromText('POINT("+str(float(msg['longitude']))+" "+str(float(msg['latitude']))+")'),"' WHERE field1='"+str(msg['UserID'])+"' AND time_stamp < '"+str(time)+"';")
          elif msg['type']==2:
        ....
        ....
        ....

问题是，在运行脚本6小时后，它会插入5分钟的文件数据！我怀疑数据是作为块而不是行传输到日志文件中，但我真的无法弄清楚如何修复它以使其更像数据库中的实时数据。

Answer 1

您是否考虑过使用psycopg2的executemany？来自this的简单示例回答：

namedict = ({"first_name":"Joshua", "last_name":"Drake"},
            {"first_name":"Steven", "last_name":"Foo"},
            {"first_name":"David", "last_name":"Bar"})

cur = conn.cursor()
cur.executemany("""INSERT INTO bar(first_name,last_name) VALUES (%(first_name)s, %(last_name)s)""", namedict)

读取作为块插入的实时数据

1 个答案: