我有一个包含850000个别更新语句的文本文件,更新了包含1200万条记录的表。这种查询的一个例子是:
update bag.pand
set mutatiedatum = to_date('04-03-2013 10:03:48','DD-MM-YYYY HH:MI:SS')
where key='0321100000015282_2013022600000000_N_0'
;
'key'在其上有唯一索引。
我在9.1 postgres / postgis数据库上使用psql.exe -q -1 -f'/ path / to / sql / file'执行该文件。
执行速度极慢:每秒约14-20个语句。
当我解释分析一个单独的陈述时,它足够快:
Update on pand (cost=0.00..6.12 rows=1 width=814) (actual time=0.101..0.101 rows=0 loops=1)
-> Index Scan using idx_pand_key on pand (cost=0.00..6.12 rows=1 width=814) (actual time=0.093..0.093 rows=0 loops=1)
Index Cond: ((key)::text = '0321100000015282_2013022600000000_N_0'::text)
Total runtime: 0.237 ms
日志文件不包含警告或错误。
据我所知,通过使用-f,所有850000语句都在一个事务中执行。这可能是缓慢的原因吗?还有其他解决方案或提示可以加快速度吗?
答案 0 :(得分:1)
如果所有更新具有相同的形状,您可以将它们加载到临时表中并使用它来更新bag.pand
表一批,如下所示:
CREATE TABLE bag.mutaties
( zkey varchar NOT NULL PRIMARY KEY
, zdate timestamp NOT NULL
);
COPY bag.mutaties(zkey,zdate) FROM 'the_big_file';
update bag.pand dst
FROM bag.mutaties src
set mutatiedatum = src.zdate
WHERE dst.key = src.zkey
;
以上假设日期为ISO'yyyy-mm-dd hh:mm:ss'格式。
如果您无法使用此日期格式创建数据文件,则可以将现有的“dd-mm-yyyy”日期读入字符串并在更新语句中进行转换(类似于您在行中的行)时间更新:
CREATE TABLE bag.mutaties
( zkey varchar NOT NULL PRIMARY KEY
, dutchdate varchar NOT NULL
);
COPY bag.mutaties(zkey,dutchdate) FROM 'the_big_file';
update bag.pand dst
FROM bag.mutaties src
set mutatiedatum = to_date(src.dutchdate, 'DD-MM-YYYY HH:MI:SS')
WHERE dst.key = src.key
;
EXTRA BONUS UPDATE :
-- This will read in the existing SQL-SCRIPT (!!)
-- and transform it into a table with {key,datetimestamp}
CREATE TABLE bag.tekstmutaties
( id SERIAL NOT NULL PRIMARY KEY
, typ INTEGER NOT NULL DEFAULT 0
, num INTEGER NOT NULL DEFAULT 0
, tekst varchar
);
-- Read in the existing script file
-- (this needs to be in /tmp/ to avoid permission problems)
COPY bag.tekstmutaties(tekst) FROM '/tmp/bagmut.txt';
-- Remove bagger
delete from bag.tekstmutaties
where LEFT(tekst,1) NOT IN ( 's' , 'w')
;
-- Extract the timestamp
UPDATE bag.tekstmutaties
SET typ = 1
, tekst = regexp_replace( tekst, E' *set.*to_date..', '', 'ig')
WHERE LEFT(tekst,1) IN ( 's')
;
UPDATE bag.tekstmutaties
SET tekst = LEFT( tekst , 19)
WHERE typ = 1
;
-- Extract the key
UPDATE bag.tekstmutaties
SET typ = 2
, tekst = regexp_replace( tekst, E' *where key..', '', 'ig')
WHERE LEFT(tekst,1) IN ( 'w')
;
UPDATE bag.tekstmutaties
SET tekst = regexp_replace( tekst , '[^0-9A-Z_]' , '' , 'g' )
WHERE typ = 2
;
-- number the records
UPDATE bag.tekstmutaties
SET num = id
WHERE TYP=1
;
-- number the records
UPDATE bag.tekstmutaties uu
SET num = src.val
FROM (
SELECT id, lag(id) OVER (ORDER BY id) AS val
FROM bag.tekstmutaties
) src
WHERE uu.TYP=2
AND src.id = uu.id
;
SELECT * FROM bag.tekstmutaties ORDER BY id;
-- The final table with the {key,timestamp} pairs
CREATE TABLE bag.mutaties
( zkey varchar NOT NULL PRIMARY KEY
, zdate timestamp NOT NULL
);
-- Fill it with self-join of teksttable
INSERT INTO bag.mutaties (zkey, zdate)
SELECT k.tekst AS zkey
, to_date(d.tekst, E'DD-MM-YYYY HH:MI:SS' ) AS zdate
FROM bag.tekstmutaties k
JOIN bag.tekstmutaties d ON k.num = d.num
WHERE k.typ=2
AND d.typ = 1
;
SELECT * FROM bag.mutaties;
-- now **after verification** you can use the mutaties-table
-- to batch-update the bag.pand table
EXPLAIN ANALYZE
update bag.pand dst
FROM mutaties src
set mutatiedatum = src.zdate
WHERE dst.key = src.key
;