Question

我有很多桌子，很少有桌子有大约2000万条记录。现在我在向UI显示结果之前对这些表进行一些计算。

为此，我创建了存储过程。在存储过程中，我使用temprorary表来存储选择查询记录，进行连接，进行处理然后返回结果。

我写了这样的查询

Insert INTO A
SELECT * from B JOIN c ....

现在我的选择查询不需要花时间（我已经使用explain扩展优化了它）但是我的Insert into需要花费很多时间，因为select的输出是百万。我的表A是临时的。

此后我还在这张A桌上做了一些处理。

我的查询是否有一种方法可以跳过插入此插入内容。我可以将结果放在一个单独的表变量（如果有的话）中，然后进行处理，而不是在temprorary中插入所有这些，然后再对它进行一些处理。

在此处添加我的存储过程

DROP PROCEDURE IF EXISTS 6_4_1n2_PortUtil_temp; DELIMITER |

CREATE PROCEDURE 6_4_1n2_PortUtil_temp(utilType VARCHAR(100),service varchar(5000),p_networkType VARCHAR(20),inputCity varchar(5000),inputNodeName varchar(5000), startTime timestamp,endTime timestamp)
BEGIN
DROP TEMPORARY TABLE IF EXISTS TEMP_SERVICE_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL_1;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEName_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEANDIF_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_ROUTERTRAFFIC_VLANPRT_SCALE1_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_TRAFFIC_TBL;
DROP TEMPORARY TABLE IF EXISTS TRAFFIC_TBL;
DROP TEMPORARY TABLE IF EXISTS FINAL_FETCH;

SELECT now();

CREATE TEMPORARY TABLE TEMP_SERVICE_TBL(nodeName varchar(256),NodeNumber int) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEName_TBL(nodeName varchar(256),NodeNumber int) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEIF_TBL(NodeNumber int,IfIndex INTEGER,IfSpeed FLOAT,IfDescr VARCHAR(100),IfAlias VARCHAR(100)) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEIF_TBL_1(NodeNumber int,IfIndex INTEGER,IfSpeed FLOAT,IfDescr VARCHAR(100),IfAlias VARCHAR(100)) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEANDIF_TBL(NodeName Varchar(256),NodeNumber int,IfIndex INTEGER,IfSpeed FLOAT,IfDescr VARCHAR(100),IfAlias VARCHAR(100),PortID BIGINT(20) DEFAULT 0) ENGINE INNODB;

CREATE TEMPORARY TABLE TRAFFIC_TBL(
    PortID                  BIGINT(20),
    NodeName        VARCHAR(100),
    IfDescr         VARCHAR(100),
    IfSpeed         VARCHAR(100),
    InErrPkts       BIGINT(20),
    RcvOctets       BIGINT(20),
    TxOctets        BIGINT(20),
    Time_1          TIMESTAMP) ENGINE INNODB;

CREATE TEMPORARY TABLE TEMP_TRAFFIC_TBL(
        PortID          BIGINT(20), 
    maxTrafficValueIn   BIGINT(20) DEFAULT 0,
    maxOutTrafficValueOut   BIGINT(20) DEFAULT 0,
    avgTrafficValueIn   BIGINT(20) DEFAULT 0,
    avgTrafficValueOut  BIGINT(20) DEFAULT 0,
    CRCError        BIGINT(20) DEFAULT 0,
    UpTime          INTEGER DEFAULT 0,
    Reliablity      INTEGER DEFAULT 0,
    AvgUtilIn       float DEFAULT 0,
    AvgUtilOut      float DEFAULT 0,
    PeakUtilIn      float DEFAULT 0, 
    PeakUtilOut         float DEFAULT 0,
    ThresholdExceed     INTEGER DEFAULT 0,
    inPeakTime      timestamp DEFAULT '0000-00-00 00:00:00',
    outPeakTime         timestamp DEFAULT '0000-00-00 00:00:00') ENGINE INNODB;

SET @where = '';
IF service='ALL'
THEN

    SET @where = '';
ELSE
set @a=1;
set @like="";
select REPLACE(SUBSTRING(SUBSTRING_INDEX(service, ',', @a),LENGTH(SUBSTRING_INDEX(service, ',', @a -1)) + 1),',','') into @service;
while(@service != "")
DO
    IF(@like = "")
    THEN
        SET @like = CONCAT("NodeName like '%",SUBSTRING(@service,2,3),"%'");
    ELSE
        SET @like = CONCAT(@like," or NodeName like '%",SUBSTRING(@service,2,3),"%'");
    END IF;
    set @a=@a+1;
    select REPLACE(SUBSTRING(SUBSTRING_INDEX(service, ',', @a),LENGTH(SUBSTRING_INDEX(service, ',', @a -1)) + 1),',','') into @service;
END WHILE;
SET @where = CONCAT(" where",@like);
END IF;


Set @where2 = '';
IF inputCity='ALL'
THEN
        set @where2 = '';
ELSE
    set @where2 = CONCAT(' and substring(NodeName,1,3) in (',inputCity,')'); 
END IF;


SET @where3 = '';
IF inputNodeName='ALL'
THEN
       set @where3 = '';
ELSE
    set @where3 = CONCAT(' and NodeName in (',inputNodeName,')');   
END IF;

SET @query1 := CONCAT("INSERT INTO TEMP_NODEName_TBL SELECT distinct NodeName,NodeNumber from NODE_TBL",@where, @where2, @where3);

SELECT @query1;

PREPARE statement1 from @query1;
EXECUTE statement1;
DEALLOCATE Prepare statement1;

CREATE INDEX n1 ON TEMP_NODEName_TBL(NodeNumber);
CREATE INDEX i1 ON TEMP_NODEIF_TBL(NodeNumber,IfIndex);
CREATE INDEX portIDIndex1 on TEMP_NODEANDIF_TBL(PortID);

SET @where4 = '';
IF (utilType='ALL')
THEN
    SET @where = '';
ELSE 
    If (utilType = "'AESI-IN'")
    THEN
        SET @where4 = " where IfAlias like '%AESI-IN%'";
    ELSE 
        SET @where4 = " where IfAlias NOT like '%AESI-IN%'";
    END IF;
END IF;




CREATE INDEX i2 ON TEMP_NODEIF_TBL_1(NodeNumber,IfIndex);
CREATE INDEX i3 ON TEMP_NODEANDIF_TBL(NodeNumber,IfIndex);

SET @where5 = ''; 
IF(p_networkType != 'ALL')
THEN
set @r1= SUBSTRING(p_networkType,2,3);
set @r2= SUBSTRING(p_networkType,8,3);
set @r3= SUBSTRING(p_networkType,14,3);
    SET @where5 = CONCAT(" and IfAlias like '%",@r1,"%'");

    if(@r2 != "")
    THEN
        SET @where5 = CONCAT(" and IfAlias like '%",@r2,"%'");

        IF(@r3 != "")
        THEN
            SET @where5 = CONCAT(" and IfAlias like '%",@r3,"%'");
        END IF;
    END IF;
END IF;

SET @query2 := CONCAT("INSERT INTO TEMP_NODEANDIF_TBL(NodeName,NodeNumber,IfIndex,IfSpeed,IfDescr,IfAlias)  SELECT distinct b.NodeName, a.NodeNumber, a.IfIndex,a.IfSpeed,a.IfDescr,a.IfAlias from NODEIF_TBL a JOIN TEMP_NODEName_TBL b ON a.NodeNumber = b.NodeNumber ", @where4, @where5);

SELECT @query2; 
PREPARE statement1 from @query2;
EXECUTE statement1;
DEALLOCATE Prepare statement1;

SELECT "DROP TEMPORARY TABLES";

DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL_1;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEName_TBL;

update TEMP_NODEANDIF_TBL a,VLANPRT_TBL b  set a.PortID = PrtID  where a.NodeNumber = b.NodeID and a.IfIndex = b.IfIndex;

SELECT "Update Temporary tables";

delete from TEMP_NODEANDIF_TBL where PortID = 0;

SELECT now();

INSERT INTO TRAFFIC_TBL 
            SELECT a.PortID,NodeName,IfDescr,IfSpeed,InErrPkts,RcvOctets,TxOctets,Time_1 FROM ROUTERTRAFFIC_VLANPRT_SCALE1_TBL a JOIN TEMP_NODEANDIF_TBL b ON a.PortID = b.PortID  where Time_1>startTime and Time_1<endTime;

SELECT now();

DROP TEMPORARY TABLE IF EXISTS EXCEED_COUNT;
CREATE TEMPORARY TABLE EXCEED_COUNT (PortID BIGINT(20), Exceed INTEGER);

INSERT INTO EXCEED_COUNT
        select PortID,count(RcvOctets) from TRAFFIC_TBL where (RcvOctets/(IfSpeed*10)>70 or TxOctets/(IfSpeed*10)>70) group by PortID;

INSERT INTO TEMP_TRAFFIC_TBL (PortID, maxTrafficValueIn,maxOutTrafficValueOut,avgTrafficValueIn,avgTrafficValueOut,CRCError,AvgUtilIn,AvgUtilOut,PeakUtilIn,PeakUtilOut)
    SELECT  PortID,max(RcvOctets),max(TxOctets),avg(RcvOctets),avg(TxOctets), sum(InErrPkts),
    IF((IfSpeed=0),"0",(avg(RcvOctets)/(IfSpeed*10))),
    IF((IfSpeed=0),"0",(avg(TxOctets)/(IfSpeed*10))),
    IF((IfSpeed=0),"0",(max(RcvOctets)/(IfSpeed*10))),
    IF((IfSpeed=0),"0",(max(TxOctets)/(IfSpeed*10)))
        from TRAFFIC_TBL group by PortID;

CREATE INDEX portIDIndex2 on TEMP_TRAFFIC_TBL(PortID);
CREATE INDEX portIDIndex3 on EXCEED_COUNT(PortID);

UPDATE TEMP_TRAFFIC_TBL A JOIN TRAFFIC_TBL B ON A.PortId=B.PortId SET inPeakTime=B.Time_1 where B.RcvOctets=A.maxTrafficValueIn;
UPDATE TEMP_TRAFFIC_TBL A JOIN TRAFFIC_TBL B ON A.PortId=B.PortId SET outPeakTime=B.Time_1 where B.TxOctets=A.maxOutTrafficValueOut;

UPDATE TEMP_TRAFFIC_TBL A JOIN EXCEED_COUNT B ON (A.PortID = B.PortID)
    set ThresholdExceed = Exceed;

SELECT substring(NodeName,5,3) as ServiceType,
CASE
WHEN IfAlias like '%SWH%' THEN "Trunk"
WHEN IfAlias like '%AES%' THEN "Backbone"
WHEN IfAlias like '%RTR%' THEN "Back-to-Back"
ELSE "-"
END ,
NodeName,IfDescr,ROUND(maxTrafficValueIn/1000,2),ROUND(maxOutTrafficValueOut/1000,2),ROUND(avgTrafficValueIn/1000,2),ROUND(avgTrafficValueOut/1000,2),ROUND(CRCError/1000,2),0,0,ROUND(AvgUtilIn,2),ROUND(AvgUtilOut,2),ROUND(PeakUtilIn,2),ROUND(PeakUtilOut,2),ThresholdExceed,inPeakTime,outPeakTime  from TEMP_TRAFFIC_TBL a ,TEMP_NODEANDIF_TBL b where a.PortID = b.PortID ;

SELECT now(); 

END |
DELIMITER ;

我的ROUTERTRAFFIC_VLANPRT_SCALE1_TBL包含大约2000万条记录和NodeIF_TBL大约10万条记录。我的VLANPRT_TBL也包含大约1万条记录。此外，我在time_1上设置了BTree索引，以便大表上的连接不需要时间

Answer 1

不需要在查询的输出中存储查询的输出，以便在另一个查询中使用它。你可以这样做：

Select t1.* from (select * from A where condition1)as t1  where condition2

以下是我必须执行的查询示例。

select avg(v1) as v1avg ,start_date_time as time, timekey as time_key from (select ATable.*, ROUND(UNIX_TIMESTAMP(start_date_time)/(60*60)) as timekey  from ATable where start_date_time between '2014-01-01 00:00:00' and '2014-01-10 00:00:00')as t1 group by timekey;

Answer 2

临时表：

临时表从内存开始，如果超过某个限制，它们将被写入磁盘。如果选择产生数百万，并且您将这些数百万写入磁盘那么这就是您的问题。

需要拥有数百万条记录的临时表通常是做错事的标志。也许您应该在将数据插入临时表进行处理之前进一步过滤数据。

磁盘I / O颠簸：

我们有几个INSERT个案例：

INSERT INTO ... VALUES
INSERT INGORE INTO ... SELECT
INSERT INTO ... SELECT

对于1.每个插入都有自己的事务，并且单独写入磁盘意味着许多微小的I / O操作。

对于2.同样适用，除了从磁盘读取数据（稍微多一些I / O操作）。

对于3.整个操作是一个大事务，意味着在事务的最后（提交）有一个大磁盘I / O.更常见的情况是，大事务中包含的信息不能完全保存在内存中，因此它会在最终提交之前临时写入磁盘，然后从那里复制到应该驻留的指定区域。

在所有3种情况下，发生磁盘I / O抖动，为了防止这种情况，您可以将小型操作分组为平均大小的操作，并将大量操作分解为平均大小的操作。

对于1.您需要在事务中包装每N个插入。

对于2.和3.您需要LIMIT SELECT子句到N并重复INSERT INTO ... SELECT并添加OFFSET N，直到您完成所有记录。

如果您使用脚本语言来运行查询，则要么相当容易。

在MySQL中插入需要花费很多时间

2 个答案:

临时表：

磁盘I / O颠簸：