批量插入数百万条记录,而不是逐行插入表格

时间:2017-07-07 02:40:07

标签: mysql sql database batch-processing

问题: **目前我的插入过程非常慢,将数据插入数据库需要很长时间,而不是逐行插入我希望实现批量插入,我怎样才能实现相同的**? 代码:这是表的架构:

DROP PROCEDURE如果存在dorepeat;

DELIMITER ;;

CREATE PROCEDURE dorepeat(IN p1 INT, IN p2 INT, IN p3 INT, IN p4 INT, IN p5 INT, IN p6 INT, IN p7 INT, IN p8 INT, IN p9 INT, IN P10 INT)
BEGIN
DECLARE x INT DEFAULT 0;
DECLARE url VARCHAR(255) DEFAULT '';
DECLARE id INT DEFAULT NULL;
DECLARE y INT DEFAULT 0;
DECLARE z INT DEFAULT 0;
DECLARE k INT DEFAULT 0;
DECLARE l INT DEFAULT 0;
DECLARE m INT DEFAULT 0;
DECLARE n INT DEFAULT 0;
DECLARE o INT DEFAULT 0;
DECLARE tagId INT DEFAULT NULL;
DECLARE accountId INT DEFAULT NULL;
DECLARE imageId INT DEFAULT NULL;
DECLARE campaignId INT DEFAULT NULL;
DECLARE hashId INT DEFAULT NULL;
DECLARE productId INT DEFAULT NULL;
DECLARE userId INT DEFAULT NULL;

INSERT INTO `auth_user`(password,last_login,is_superuser,username,first_name,last_name,email,is_staff,is_active,date_joined) VALUES ('12',now(),1,UUID(),'first','last','email',1,1,now());
SET userId = LAST_INSERT_ID();


INSERT INTO `server_tag`(name) VALUES('123');
SET tagId = LAST_INSERT_ID();

SET o = 0; REPEAT 
    INSERT INTO `server_account`(`name`,`key`,`account_id`) VALUES (LEFT(MD5(o), 30), '1', 1);
    SET accountId = LAST_INSERT_ID();

    SET n = 0; REPEAT 
        INSERT INTO `server_campaign`(name,app_id,app_name,`access_key`,`secret_key`,`meta_config`,`full_text_searchable`,`updated`,`created`,`account_id`) 
        VALUES 
        (LEFT(MD5(n), 30), 
        1,
         '123', 
         '123',
         '123', 
         '123', 
         1, 
         now(), 
         now(), 
         accountId
         );
        SET campaignId = LAST_INSERT_ID();

        SET m = 0; REPEAT 
            INSERT INTO `server_hashtag`(NAME, status) VALUES (LEFT(MD5(m), 50), 1);
            SET hashId = LAST_INSERT_ID();

            SET l = 0; REPEAT
                SET url = CONCAT('http://example.com/', MD5(CONCAT(hashId, ':', l)));
                INSERT INTO `server_image`(hex, url, low_url, thumbnail_url, box, username, link, posted, updated, status, score, hashtag_id)
                VALUES (
                SHA1(url),
                url,
                CONCAT('a', hashId),
                CONCAT('b', hashId),
                CONCAT('c', hashId),
                CONCAT('d', hashId),
                CONCAT('e', hashId),
                NOW(),
                NOW(),
                1,
                CONCAT('f', hashId),
                hashId
                );
                SET imageId = LAST_INSERT_ID();

                SET k = 0; REPEAT 
                    INSERT INTO `server_product`(im_name, meta) VALUES (LEFT(UUID(), 1000000), 1);
                    SET productId = LAST_INSERT_ID();


                    INSERT INTO `server_image_product`(campaign_id, image_id, product_id) VALUES (campaignId,imageId,productId);
                    INSERT INTO `server_image_status`(status,updated,campaign_id,image_id,user_id) VALUES(2,now(),campaignId,imageId,userId);
                    INSERT INTO `server_image_tag`(campaign_id,image_id,tag_id) VALUES (campaignId, imageId, tagId);


                    SET k = k + 1;
                UNTIL k >= p7 END REPEAT;

                SET l = l + 1;
            UNTIL l >= p6 END REPEAT;

            SET m = m + 1;
        UNTIL m >= p5 END REPEAT;

        SET n = n + 1;
    UNTIL n >= p4 END REPEAT;

    SET o = o + 1;
UNTIL o >= p3 END REPEAT;

END;;

DELIMITER ;

SET FOREIGN_KEY_CHECKS = 0;
TRUNCATE TABLE `auth_user`;
TRUNCATE TABLE `server_tag`;
TRUNCATE TABLE `server_image`; 
TRUNCATE TABLE `server_hashtag`; 
TRUNCATE TABLE `server_product`; 
TRUNCATE TABLE `server_account`;
TRUNCATE TABLE `server_campaign`;
TRUNCATE TABLE `server_image_product`;
TRUNCATE TABLE `server_image_status`;
TRUNCATE TABLE `server_image_tag`;
SET FOREIGN_KEY_CHECKS = 1;


CALL dorepeat(1, 1, 100, 100, 100, 1000000, 1000000, 100, 100, 100);

问题: **目前我的插入过程非常慢,将数据插入数据库需要很长时间,而不是逐行插入我希望实现批量插入,我怎样才能实现相同的**?

0 个答案:

没有答案