我有一个大型SQL查询（mysql），用于填充CRM系统中的记录表。
这种方法运行良好，速度非常快，约有4000条记录。现在达到15,500，它的运行速度非常缓慢。大约需要70秒才能返回数据。
我尝试过添加一些索引，但成效有限。有什么建议吗？
查询是：
SELECT SQL_NO_CACHE SQL_CALC_FOUND_ROWS
s.*,
ca2.address_postcode,
ca2.resident_status
FROM
(
SELECT
a.id,
c.name_first,
c.name_last,
a.created as app_created,
a.edited,
a.comments as custcomment,
c.name_company,
a.loan_amount,
a.product_type,
ap.packager as placed_with,
a.introducer_id,
a.status,
c.contact_number,
c.email,
a.database,
bd.legal_structure,
(
SELECT
ca1.id
FROM
cl_customer_address AS ca1
WHERE
ca1.customer_id = a.customer_id AND
ca1.deleted = "0000-00-00 00:00:00"
ORDER BY
ca1.created DESC
LIMIT
1
) AS address_id
FROM
cl_application AS a
LEFT JOIN
cl_application_business_details as bd on bd.id = a.id
LEFT JOIN
cl_customer AS c ON c.id = a.customer_id AND c.deleted = c.deleted
LEFT JOIN
cl_application_packager AS ap ON ap.application_id = a.id AND ap.status != "Declined" AND ap.deleted = "0000-00-00 00:00:00"
WHERE
(a.deleted = "0000-00-00 00:00:00")
GROUP BY
a.id
) AS s
LEFT JOIN
cl_customer_address AS ca2 ON ca2.id = s.address_id AND ca2.deleted = ca2.deleted
WHERE
(ca2.deleted = ca2.deleted OR ca2.deleted IS NULL)
ORDER BY
app_created DESC
LIMIT
0, 100;
Time: 70.382
Rows: 100
表格描述如下：
CREATE TABLE cl_application (
id int(11) NOT NULL AUTO_INCREMENT,
customer_id int(11) NOT NULL,
product_type tinytext COLLATE utf8_unicode_ci,
introducer_id int(11) NOT NULL,
loan_amount double NOT NULL,
loan_purpose tinytext COLLATE utf8_unicode_ci NOT NULL,
comments text COLLATE utf8_unicode_ci NOT NULL,
security_value double NOT NULL,
property_address_1 tinytext COLLATE utf8_unicode_ci NOT NULL,
property_address_2 tinytext COLLATE utf8_unicode_ci NOT NULL,
property_town_city tinytext COLLATE utf8_unicode_ci NOT NULL,
property_postcode varchar(8) COLLATE utf8_unicode_ci NOT NULL,
property_country tinytext COLLATE utf8_unicode_ci NOT NULL,
application_source tinytext COLLATE utf8_unicode_ci NOT NULL,
`status` enum('WK - Working Lead','APP - Application Taken','ISS - Pack Issued','HOT - Head Of Terms Sent','APU - Application underway','OFI - Offer Issued','DIP - Deal in Progress','DUS - Declined Unsecured/Trying Secured','DUG - Declined Unsecured/Trying Guarantor','COM - Completed Awaiting Payment','PAC - Paid and Completed','TD - Turned Down','DUP - Duplicate application') COLLATE utf8_unicode_ci NOT NULL DEFAULT 'WK - Working Lead',
multi_stage_status text COLLATE utf8_unicode_ci NOT NULL,
owner_id int(11) NOT NULL,
token tinytext COLLATE utf8_unicode_ci NOT NULL,
cache_keyword text COLLATE utf8_unicode_ci NOT NULL,
old_id int(11) NOT NULL,
placed_with tinytext COLLATE utf8_unicode_ci NOT NULL,
submitted datetime NOT NULL,
`database` enum('LV','TD','CD','UL') COLLATE utf8_unicode_ci NOT NULL DEFAULT 'LV',
snoozed datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
created datetime NOT NULL,
edited datetime NOT NULL,
deleted datetime NOT NULL,
PRIMARY KEY (id),
FULLTEXT KEY cache_keyword (cache_keyword)
) ;
CREATE TABLE cl_application_packager (
id int(11) NOT NULL AUTO_INCREMENT,
application_id int(11) NOT NULL,
packager tinytext COLLATE utf8_unicode_ci,
packager_id int(11) DEFAULT NULL,
amount double DEFAULT NULL,
`status` enum('In Progress','Declined','Accepted','Completed') COLLATE utf8_unicode_ci DEFAULT 'In Progress',
commision double DEFAULT NULL,
created datetime NOT NULL,
edited datetime NOT NULL,
deleted datetime NOT NULL,
PRIMARY KEY (id),
KEY application_id (deleted)
) ;
CREATE TABLE cl_application_business_details (
id int(11) NOT NULL AUTO_INCREMENT,
company_number tinytext COLLATE utf8_unicode_ci,
legal_structure enum('Ltd','LLP','Partnership','Sole Trader') COLLATE utf8_unicode_ci DEFAULT NULL,
incorporated date DEFAULT NULL,
created datetime NOT NULL,
edited datetime NOT NULL,
deleted datetime NOT NULL,
PRIMARY KEY (id)
) ;
CREATE TABLE cl_customer (
id int(11) NOT NULL AUTO_INCREMENT,
title enum('Mr','Mrs','Ms','Miss') COLLATE utf8_unicode_ci NOT NULL,
name_first tinytext COLLATE utf8_unicode_ci NOT NULL,
name_last tinytext COLLATE utf8_unicode_ci NOT NULL,
dob date NOT NULL,
marital_status tinytext COLLATE utf8_unicode_ci NOT NULL,
name_company tinytext COLLATE utf8_unicode_ci NOT NULL,
email tinytext COLLATE utf8_unicode_ci NOT NULL,
alt_email tinytext COLLATE utf8_unicode_ci,
contact_number tinytext COLLATE utf8_unicode_ci NOT NULL,
home_phone_number tinytext COLLATE utf8_unicode_ci NOT NULL,
work_phone_number tinytext COLLATE utf8_unicode_ci NOT NULL,
created datetime NOT NULL,
edited datetime NOT NULL,
deleted datetime NOT NULL,
PRIMARY KEY (id)
) ;
CREATE TABLE cl_customer_address (
id int(11) NOT NULL AUTO_INCREMENT,
customer_id int(11) NOT NULL,
application_id int(11) NOT NULL,
house_name tinytext COLLATE utf8_unicode_ci NOT NULL,
house_number tinytext COLLATE utf8_unicode_ci NOT NULL,
address_line_1 tinytext COLLATE utf8_unicode_ci NOT NULL,
address_line_2 tinytext COLLATE utf8_unicode_ci NOT NULL,
address_town tinytext COLLATE utf8_unicode_ci NOT NULL,
address_postcode tinytext COLLATE utf8_unicode_ci NOT NULL,
moved_in date NOT NULL,
vacated date NOT NULL DEFAULT '0000-00-00',
ptcabs tinytext COLLATE utf8_unicode_ci NOT NULL,
resident_status tinytext COLLATE utf8_unicode_ci NOT NULL,
created datetime NOT NULL,
edited datetime NOT NULL,
deleted datetime NOT NULL,
PRIMARY KEY (id),
KEY customer_id (customer_id,deleted)
) ;
mysql> describe cl_application
-> ;
+--------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+-----+---------------------+----------------+
| Field | Type | Null | Key | Default | Extra |
+--------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+-----+---------------------+----------------+
| id | int(11) | NO | PRI | NULL | auto_increment |
| customer_id | int(11) | NO | | NULL | |
| product_type | tinytext | YES | | NULL | |
| introducer_id | int(11) | NO | | NULL | |
| loan_amount | double | NO | | NULL | |
| loan_purpose | tinytext | NO | | NULL | |
| comments | text | NO | | NULL | |
| security_value | double | NO | | NULL | |
| property_address_1 | tinytext | NO | | NULL | |
| property_address_2 | tinytext | NO | | NULL | |
| property_town_city | tinytext | NO | | NULL | |
| property_postcode | varchar(8) | NO | | NULL | |
| property_country | tinytext | NO | | NULL | |
| application_source | tinytext | NO | | NULL | |
| status | enum('WK - Working Lead','APP - Application Taken','ISS - Pack Issued','HOT - Head Of Terms Sent','APU - Application underway','OFI - Offer Issued','DIP - Deal in Progress','DUS - Declined Unsecured/Trying Secured','DUG - Declined Unsecured/Trying Guarantor','COM - Completed Awaiting Payment','PAC - Paid and Completed','TD - Turned Down','DUP - Duplicate application') | NO | | WK - Working Lead | |
| multi_stage_status | text | NO | | NULL | |
| owner_id | int(11) | NO | | NULL | |
| token | tinytext | NO | | NULL | |
| cache_keyword | text | NO | MUL | NULL | |
| old_id | int(11) | NO | | NULL | |
| placed_with | tinytext | NO | | NULL | |
| submitted | datetime | NO | | NULL | |
| database | enum('LV','TD','CD','UL') | NO | | LV | |
| snoozed | datetime | NO | | 0000-00-00 00:00:00 | |
| created | datetime | NO | | NULL | |
| edited | datetime | NO | | NULL | |
| deleted | datetime | NO | | NULL | |
+--------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+-----+---------------------+----------------+
27 rows in set (0.00 sec)
-
mysql> describe cl_application_business_details;
+-----------------+-----------------------------------------------+------+-----+---------+----------------+
| Field | Type | Null | Key | Default | Extra |
+-----------------+-----------------------------------------------+------+-----+---------+----------------+
| id | int(11) | NO | PRI | NULL | auto_increment |
| company_number | tinytext | YES | | NULL | |
| legal_structure | enum('Ltd','LLP','Partnership','Sole Trader') | YES | | NULL | |
| incorporated | date | YES | | NULL | |
| created | datetime | NO | | NULL | |
| edited | datetime | NO | | NULL | |
| deleted | datetime | NO | | NULL | |
+-----------------+-----------------------------------------------+------+-----+---------+----------------+
7 rows in set (0.00 sec)
-
mysql> describe cl_customer;
+-------------------+------------------------------+------+-----+---------+----------------+
| Field | Type | Null | Key | Default | Extra |
+-------------------+------------------------------+------+-----+---------+----------------+
| id | int(11) | NO | PRI | NULL | auto_increment |
| title | enum('Mr','Mrs','Ms','Miss') | NO | | NULL | |
| name_first | tinytext | NO | | NULL | |
| name_last | tinytext | NO | | NULL | |
| dob | date | NO | | NULL | |
| marital_status | tinytext | NO | | NULL | |
| name_company | tinytext | NO | | NULL | |
| email | tinytext | NO | | NULL | |
| alt_email | tinytext | YES | | NULL | |
| contact_number | tinytext | NO | | NULL | |
| home_phone_number | tinytext | NO | | NULL | |
| work_phone_number | tinytext | NO | | NULL | |
| created | datetime | NO | | NULL | |
| edited | datetime | NO | | NULL | |
| deleted | datetime | NO | | NULL | |
+-------------------+------------------------------+------+-----+---------+----------------+
15 rows in set (0.00 sec)
-
mysql> describe cl_application_packager;
+----------------+-------------------------------------------------------+------+-----+-------------+----------------+
| Field | Type | Null | Key | Default | Extra |
+----------------+-------------------------------------------------------+------+-----+-------------+----------------+
| id | int(11) | NO | PRI | NULL | auto_increment |
| application_id | int(11) | NO | | NULL | |
| packager | tinytext | YES | | NULL | |
| packager_id | int(11) | YES | | NULL | |
| amount | double | YES | | NULL | |
| status | enum('In Progress','Declined','Accepted','Completed') | YES | | In Progress | |
| commision | double | YES | | NULL | |
| created | datetime | NO | | NULL | |
| edited | datetime | NO | | NULL | |
| deleted | datetime | NO | MUL | NULL | |
+----------------+-------------------------------------------------------+------+-----+-------------+----------------+
10 rows in set (0.00 sec)
mysql> describe cl_customer_address;
+------------------+----------+------+-----+------------+----------------+
| Field | Type | Null | Key | Default | Extra |
+------------------+----------+------+-----+------------+----------------+
| id | int(11) | NO | PRI | NULL | auto_increment |
| customer_id | int(11) | NO | MUL | NULL | |
| application_id | int(11) | NO | | NULL | |
| house_name | tinytext | NO | | NULL | |
| house_number | tinytext | NO | | NULL | |
| address_line_1 | tinytext | NO | | NULL | |
| address_line_2 | tinytext | NO | | NULL | |
| address_town | tinytext | NO | | NULL | |
| address_postcode | tinytext | NO | | NULL | |
| moved_in | date | NO | | NULL | |
| vacated | date | NO | | 0000-00-00 | |
| ptcabs | tinytext | NO | | NULL | |
| resident_status | tinytext | NO | | NULL | |
| created | datetime | NO | | NULL | |
| edited | datetime | NO | | NULL | |
| deleted | datetime | NO | | NULL | |
+------------------+----------+------+-----+------------+----------------+
16 rows in set (0.00 sec)
解释输出：
id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY <derived2> ALL 178913936 Using filesort
1 PRIMARY ca2 eq_ref PRIMARY PRIMARY 4 s.address_id 1 Using where
2 DERIVED a ALL PRIMARY, cache_keyword 14494 Using where; Using temporary; Using filesort
2 DERIVED bd eq_ref PRIMARY PRIMARY 4 s-choiceloans.a.id 1
2 DERIVED c eq_ref PRIMARY PRIMARY 4 s-choiceloans.a.customer_id 1 Using where
2 DERIVED ap ALL 12344 Using where; Using join buffer (Block Nested Loop)
3 DEPENDENT SUBQUERY ca1 ref customer_id customer_id 9 func,const 1 Using where; Using filesort
根据答案的后续更改查询后：
答案 0 :(得分：3)
最大的疑问是内联视图（或派生表，在MySQL版本中）.MySQL将实现派生表，然后外部查询将针对它运行。
另一个嫌疑人是SELECT列表中的相关子查询。这将由外部查询返回的每一行执行。
此外，ORDER BY app_created将需要Using filesort操作（除非您运行的是可能在派生表上构建索引的更新版本的MySQL。）
有一些奇怪的谓词，例如c.deleted = c.deleted
对于deleted列中具有非NULL值的每一行，情况都是如此。这相当于c.deleted IS NOT NULL。
为此：ca2.deleted = ca2.deleted OR ca2.deleted IS NULL，这总是如此。
然而，这些都不会对性能产生太大影响。
运行EXPLAIN以查看执行计划。
缺席，这是对某些建议的第一次削减：
在c1_application_packager
CREATE INDEX c1_application_packager_IX2 ON c1_application_packager
(application_id, deleted, status, packager)
;
在cl_customer_address
CREATE INDEX c1_customer_address_IX2 ON c1_customer_address
(customer_id, deleted, created, id, address_postcode, resident_status)
;
重新编写查询以消除派生表。使用从c1_customer_address返回id的相同相关子查询将连接替换为ca2 ...
SELECT a.id
, c.name_first
, c.name_last
, a.created AS app_created
, a.edited
, a.comments AS custcomment
, c.name_company
, a.loan_amount
, a.product_type
, ap.packager AS placed_with
, a.introducer_id
, a.status
, c.contact_number
, c.email
, a.database
, bd.legal_structure
, ( SELECT ca1.id
FROM cl_customer_address ca1
WHERE ca1.customer_id = a.customer_id
AND ca1.deleted = '0000-00-00 00:00:00'
ORDER BY ca1.customer_id, ca1.deleted, ca1.created DESC
LIMIT 1
) AS address_id
, ( SELECT ca2.address_postcode
FROM cl_customer_address ca2
WHERE ca2.customer_id = a.customer_id
AND ca2.deleted = '0000-00-00 00:00:00'
ORDER BY ca2.customer_id, ca2.deleted, ca2.created DESC
LIMIT 1
) AS address_postcode
, ( SELECT ca3.resident_status
FROM cl_customer_address ca3
WHERE ca3.customer_id = a.customer_id
AND ca3.deleted = '0000-00-00 00:00:00'
ORDER BY ca3.customer_id, ca3.deleted, ca3.created DESC
LIMIT 1
) AS resident_status
FROM cl_application a
LEFT
JOIN cl_application_business_details bd
ON bd.id = a.id
LEFT
JOIN cl_customer c
ON c.id = a.customer_id
AND c.deleted IS NOT NULL
LEFT
JOIN cl_application_packager ap
ON ap.application_id = a.id
AND ap.status != 'Declined'
AND ap.deleted = '0000-00-00 00:00:00'
WHERE a.deleted = '0000-00-00 00:00:00'
GROUP BY a.created, a.id
ORDER BY a.created, a.id
SELECT列表中的那些相关子查询将针对查询返回的每一行执行，因此这将是昂贵的。
现在我们想看看我们是否可以在c1_application表上获得一个索引来帮助我们避免Using filesort操作（以满足ORDER BY和GROUP BY子句。）
CREATE INDEX c1_application_IX2 ON c1_application
(deleted, created, id)
;
查询依赖于特定于MySQL的GROUP BY行为扩展，而不是由于SELECT列表中未出现在GROUP BY中的非聚合而引发错误。如果c1_customer或c1_application_packager中存在多个“匹配”行，那么从GROUP BY操作返回的行是不确定的。
无法保证这些更改会对性能产生积极影响。 （表现可能会更糟糕。）
再次，运行EXPLAIN以查看执行计划，并从那里进行调整。
下一个大驼峰是那些相关的子查询。为了获得良好的性能，必须提供合适的索引。 （关于上文已经指定的合适覆盖指数的提议。）
下一个剪切将消除SELECT列表中的相关子查询，如果返回id中的c1_customer_address列作为获取address_postcode和{{1}的方法}，可以消除第一个相关的子查询。
关注
删除相关子查询...添加内联视图resident_status以从c1_customer_address（对于每个customer_id）获取最新创建日期，并添加另一个连接到c1_customer_address以检索具有该最新创建日期的行（对于每个顾客）。
lc内联视图引入了一个“派生表”，对于大型集合来说这可能很昂贵，但这可能比使用相关子查询更快。
lc