民间
我有桌子（带结构）
Billing_datas_new有大约5000万条记录，每天都在增加。 （呼叫详细记录）与所需的ID一起转储到此表中
CREATE TABLE IF NOT EXISTS `cctonumbers` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`countrycode_id` int(11) NOT NULL,
`parentid` int(11) NOT NULL,
`number` char(20) NOT NULL,
`dialcode` int(11) NOT NULL,
`effectivedate` datetime NOT NULL,
`route_code` tinyint(4) NOT NULL,
`active` tinyint(4) NOT NULL,
`status` tinyint(11) NOT NULL,
`createdby` int(11) NOT NULL,
`createdon` datetime NOT NULL,
`modifiedby` int(11) NOT NULL,
`modifiedon` datetime NOT NULL,
PRIMARY KEY (`id`),
KEY `effectivedate` (`effectivedate`),
KEY `parentid` (`parentid`),
KEY `number` (`number`),
KEY `dialcode` (`dialcode`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=15729 ;
CREATE TABLE IF NOT EXISTS `billing_datas_NEW` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`batch_id` int(11) NOT NULL,
`file_url` varchar(255) NOT NULL,
`switch_id` int(11) NOT NULL,
`carrierid_supplier` int(11) NOT NULL,
`technical_profileid_supplier` int(11) NOT NULL,
`carrierid_customer` int(11) NOT NULL,
`technical_profileid_customer` int(11) NOT NULL,
`billing_increment_supplier` varchar(10) NOT NULL,
`billing_increment_customer` varchar(10) NOT NULL,
`billable_duration_supplier` int(11) NOT NULL,
`call_duration_seconds` int(11) NOT NULL,
`billable_duration_customer` int(11) NOT NULL,
`destination` varchar(50) NOT NULL,
`destination_country` varchar(50) NOT NULL,
`rateplanid_supplier` int(11) NOT NULL,
`rateplanid_customer` int(11) NOT NULL,
`rate_supplier` int(11) NOT NULL,
`rate_customer` int(11) NOT NULL,
`rate_total_supplier` int(11) NOT NULL,
`rate_total_customer` int(11) NOT NULL,
`rate_effective_date_supplier` date NOT NULL,
`rate_effective_date_customer` date NOT NULL,
`call_hour` varchar(10) NOT NULL,
`sequence_number` int(11) NOT NULL,
`version` varchar(10) NOT NULL,
`record_type` varchar(1) NOT NULL,
`connection_type` varchar(16) NOT NULL,
`session_id` varchar(36) NOT NULL,
`release_cause` smallint(6) NOT NULL,
`start_time_date` datetime NOT NULL,
`answer_time_date` datetime NOT NULL,
`release_tod` datetime NOT NULL,
`greenwich_mean_time` varchar(32) NOT NULL,
`release_cause_protocol_stack` varchar(32) NOT NULL,
`binary_value_protocol_stack` smallint(6) NOT NULL,
`first_release_dialogue` varchar(1) NOT NULL,
`origination_trunkid` int(11) NOT NULL,
`voip_protocol` varchar(6) NOT NULL,
`source_number` varchar(128) NOT NULL,
`source_host_name` varchar(128) NOT NULL,
`destination_number` varchar(128) NOT NULL,
`destination_host_name` varchar(128) NOT NULL,
`callid` varchar(128) NOT NULL,
`remote_payload_ipaddress` varchar(16) NOT NULL,
`remote_payload_udpaddress` varchar(6) NOT NULL,
`local_payload_ipaddress` varchar(16) NOT NULL,
`local_payload_udpaddress` varchar(6) NOT NULL,
`codec_list` varchar(128) NOT NULL,
`ingress_packets` int(11) NOT NULL,
`egress_packets` int(11) NOT NULL,
`ingress_octets` int(11) NOT NULL,
`egress_octets` int(11) NOT NULL,
`ingress_packet_loss` int(11) NOT NULL,
`ingress_delay` int(11) NOT NULL,
`ingress_packet_jitter` int(11) NOT NULL,
`supplierid` mediumint(9) NOT NULL,
`protocol` varchar(6) NOT NULL,
`termination_source_number` varchar(128) NOT NULL,
`termination_source_host` varchar(128) NOT NULL,
`termination_destination_number` varchar(128) NOT NULL,
`termination_destination_host_name` varchar(128) NOT NULL,
`termination_callid` varchar(128) NOT NULL,
`termination_remote_payload_ipaddress` varchar(16) NOT NULL,
`termination_remote_payload_udpaddress` varchar(6) NOT NULL,
`termination_local_payload_ipaddress` varchar(16) NOT NULL,
`termination_local_payload_udpaddress` varchar(6) NOT NULL,
`termination_codec_list` varchar(128) NOT NULL,
`termination_ingress_packets` int(11) NOT NULL,
`termination_egress_packets` int(11) NOT NULL,
`termination_ingress_octets` int(11) NOT NULL,
`termination_egress_octets` int(11) NOT NULL,
`termination_ingress_packet_loss` int(11) NOT NULL,
`termination_ingress_delay` int(11) NOT NULL,
`termination_ingress_packet_jitter` int(11) NOT NULL,
`final_route_indication` varchar(1) NOT NULL,
`routing_digits` varchar(64) NOT NULL,
`call_duration` mediumint(9) NOT NULL,
`post_dial_delay` mediumint(9) NOT NULL,
`ring_time` mediumint(9) NOT NULL,
`call_duration_ms` int(11) NOT NULL,
`confid` varchar(32) NOT NULL,
`rpid` varchar(32) NOT NULL,
`route_entry_index` tinyint(4) NOT NULL,
`route_table_used` mediumint(9) NOT NULL,
`lnp_dipped` varchar(1) NOT NULL,
`ingress_lrn` varchar(32) NOT NULL,
`egress_lrn` varchar(32) NOT NULL,
`cnam_dipped` tinyint(4) NOT NULL,
`dnc_dipped` tinyint(4) NOT NULL,
`origination_device_name` varchar(15) NOT NULL,
`termination_device_name` varchar(15) NOT NULL,
`ers_dipped` varchar(1) NOT NULL,
`oli_digits` varchar(8) NOT NULL,
`routing_match` varchar(41) NOT NULL,
`termination_pdd` varchar(6) NOT NULL,
`proxy_charge_information` varchar(41) NOT NULL,
`jurisdictoinal_indication_parameter` varchar(41) NOT NULL,
`matched_digits_ers` varchar(41) NOT NULL,
`route_table_irs` varchar(5) NOT NULL,
`route_sequence_number_ers` varchar(2) NOT NULL,
`jurisdiction_type_ers` varchar(3) NOT NULL,
PRIMARY KEY (`id`),
KEY `switch_id` (`switch_id`),
KEY `carrieridsupplier` (`carrierid_supplier`),
KEY `technicalprofileidsupplier` (`technical_profileid_supplier`),
KEY `carrieridcustomer` (`carrierid_customer`),
KEY `technicalprofileidcustomer` (`technical_profileid_customer`),
KEY `destination` (`destination`),
KEY `destinationcountry` (`destination_country`),
KEY `rateplanidsupplier` (`rateplanid_supplier`),
KEY `rateplanidcustomer` (`rateplanid_customer`),
KEY `callhour` (`call_hour`),
KEY `recordtype` (`record_type`),
KEY `connectiontype` (`connection_type`),
KEY `sessionid` (`session_id`),
KEY `releasecause` (`release_cause`),
KEY `starttimedate` (`start_time_date`),
KEY `protocolstack` (`release_cause_protocol_stack`),
KEY `binaryvalue` (`binary_value_protocol_stack`),
KEY `originationtrunkid` (`origination_trunkid`),
KEY `sourcehostname` (`source_host_name`),
KEY `destinationhostname` (`destination_host_name`),
KEY `supplierid` (`supplierid`),
KEY `terminationsourcehost` (`termination_source_host`),
KEY `terminationdestinationhostname` (`termination_destination_host_name`),
KEY `callduration` (`call_duration`),
KEY `postdialdelay` (`post_dial_delay`),
KEY `batch_id` (`batch_id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=56232766 ;
CREATE TABLE IF NOT EXISTS `carriermasters` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`carriername` varchar(255) NOT NULL,
`carriershortname` varchar(20) NOT NULL,
`website` varchar(255) NOT NULL,
`phone` int(11) NOT NULL,
`fax` int(11) NOT NULL,
`email` varchar(255) NOT NULL,
`billingaddress` varchar(255) NOT NULL,
`billingcity` varchar(200) NOT NULL,
`billingstate` varchar(200) NOT NULL,
`billingcountry` varchar(200) NOT NULL,
`zipcode` int(11) DEFAULT NULL,
`description` varchar(255) NOT NULL,
`minimumseconds` int(11) NOT NULL,
`incrementseconds` int(11) NOT NULL,
`active` tinyint(4) NOT NULL DEFAULT '1',
`createdby` int(11) NOT NULL,
`createdon` datetime NOT NULL,
`modifiedby` int(11) NOT NULL,
`modifiedon` datetime NOT NULL,
`account_manager` varchar(500) NOT NULL,
`status` varchar(100) NOT NULL,
PRIMARY KEY (`id`),
KEY `carriershortname` (`carriershortname`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=306 ;
CREATE TABLE IF NOT EXISTS `technicalprofiles` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`carrier_id` int(11) NOT NULL,
`profilename` varchar(255) NOT NULL,
`active` tinyint(4) NOT NULL DEFAULT '1',
`createdby` int(50) NOT NULL,
`createdon` datetime NOT NULL,
`modifiedby` int(50) NOT NULL,
`modifiedon` datetime NOT NULL,
PRIMARY KEY (`id`),
KEY `carrier_id` (`carrier_id`),
KEY `profilename` (`profilename`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=479 ;
服务器的MY.CNF如下
# Example MySQL config file for very large systems.
#
# This is for a large system with memory of 1G-2G where the system runs mainly
# MySQL.
#
# MySQL programs look for option files in a set of
# locations which depend on the deployment platform.
# You can copy this option file to one of those
# locations. For information about these locations, see:
# http://dev.mysql.com/doc/mysql/en/option-files.html
#
# In this file, you can use all long options that a program supports.
# If you want to know which options a program supports, run the program
# with the "--help" option.
# The following options will be passed to all MySQL clients
[client]
#password = your_password
port = 3306
socket = /var/lib/mysql/mysql.sock
# Here follows entries for some specific programs
# The MySQL server
[mysqld]
port = 3306
socket = /var/lib/mysql/mysql.sock
skip-locking
key_buffer_size = 3G
max_allowed_packet = 16M
table_open_cache = 1024
sort_buffer_size = 256M
read_buffer_size = 1024M
read_rnd_buffer_size = 64M
myisam_sort_buffer_size = 4G
thread_cache_size = 8
query_cache_size = 256M
query_cache_limit = 256M
max_connections = 100
max_heap_table_size = 64M
join_buffer_size = 8M
tmp_table_size = 1024M
myisam_max_sort_file_size = 3G
slow_query_log
# Try number of CPU's*2 for thread_concurrency
thread_concurrency = 8
# Don't listen on a TCP/IP port at all. This can be a security enhancement,
# if all processes that need to connect to mysqld run on the same host.
# All interaction with mysqld must be made via Unix sockets or named pipes.
# Note that using this option without enabling named pipes on Windows
# (via the "enable-named-pipe" option) will render mysqld useless!
#
#skip-networking
# Replication Master Server (default)
# binary logging is required for replication
log-bin=mysql-bin
# required unique id between 1 and 2^32 - 1
# defaults to 1 if master-host is not set
# but will not function as a master if omitted
server-id = 1
# Replication Slave (comment out master section to use this)
#
# To configure this host as a replication slave, you can choose between
# two methods :
#
# 1) Use the CHANGE MASTER TO command (fully described in our manual) -
# the syntax is:
#
# CHANGE MASTER TO MASTER_HOST=<host>, MASTER_PORT=<port>,
# MASTER_USER=<user>, MASTER_PASSWORD=<password> ;
#
# where you replace <host>, <user>, <password> by quoted strings and
# <port> by the master's port number (3306 by default).
#
# Example:
#
# CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=3306,
# MASTER_USER='joe', MASTER_PASSWORD='secret';
#
# OR
#
# 2) Set the variables below. However, in case you choose this method, then
# start replication for the first time (even unsuccessfully, for example
# if you mistyped the password in master-password and the slave fails to
# connect), the slave will create a master.info file, and any later
# change in this file to the variables' values below will be ignored and
# overridden by the content of the master.info file, unless you shutdown
# the slave server, delete master.info and restart the slaver server.
# For that reason, you may want to leave the lines below untouched
# (commented) and instead use CHANGE MASTER TO (see above)
#
# required unique id between 2 and 2^32 - 1
# (and different from the master)
# defaults to 2 if master-host is set
# but will not function as a slave if omitted
#server-id = 2
#
# The replication master for this slave - required
#master-host = <hostname>
#
# The username the slave will use for authentication when connecting
# to the master - required
#master-user = <username>
#
# The password the slave will authenticate with when connecting to
# the master - required
#master-password = <password>
#
# The port the master is listening on.
# optional - defaults to 3306
#master-port = <port>
#
# binary logging - not required for slaves, but recommended
#log-bin=mysql-bin
#
# binary logging format - mixed recommended
#binlog_format=mixed
# Uncomment the following if you are using InnoDB tables
#innodb_data_home_dir = /var/lib/mysql
#innodb_data_file_path = ibdata1:2000M;ibdata2:10M:autoextend
#innodb_log_group_home_dir = /var/lib/mysql
# You can set .._buffer_pool_size up to 50 - 80 %
# of RAM but beware of setting memory usage too high
#innodb_buffer_pool_size = 384M
#innodb_additional_mem_pool_size = 20M
# Set .._log_file_size to 25 % of buffer pool size
#innodb_log_file_size = 100M
#innodb_log_buffer_size = 8M
#innodb_flush_log_at_trx_commit = 1
#innodb_lock_wait_timeout = 50
[mysqldump]
quick
max_allowed_packet = 100M
[mysql]
no-auto-rehash
# Remove the next comment character if you are not familiar with SQL
#safe-updates
[myisamchk]
key_buffer_size = 3G
sort_buffer_size = 512M
read_buffer = 1024M
write_buffer = 8M
[mysqlhotcopy]
interactive-timeout
mysql tuner报告如下
-- MYSQL PERFORMANCE TUNING PRIMER --
- By: Matthew Montgomery -
MySQL Version 5.1.69-log x86_64
Uptime = 1 days 9 hrs 7 min 0 sec
Avg. qps = 13
Total Questions = 1616144
Threads Connected = 2
Warning: Server has not been running for at least 48hrs.
It may not be safe to use these recommendations
To find out more information on how each of these
runtime variables effects performance visit:
http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html
Visit http://www.mysql.com/products/enterprise/advisors.html
for info about MySQL's Enterprise Monitoring and Advisory Service
SLOW QUERIES
The slow query log is enabled.
Current long_query_time = 10.000000 sec.
You have 30 out of 1616167 that take longer than 10.000000 sec. to complete
Your long_query_time seems to be fine
BINARY UPDATE LOG
The binary update log is enabled
The expire_logs_days is not set.
The mysqld will retain the entire binary log until RESET MASTER or PURGE MASTER LOGS commands are run manually
Setting expire_logs_days will allow you to remove old binary logs automatically
See http://dev.mysql.com/doc/refman/5.1/en/purge-master-logs.html
Binlog sync is not enabled, you could loose binlog records during a server crash
WORKER THREADS
Current thread_cache_size = 8
Current threads_cached = 6
Current threads_per_sec = 0
Historic threads_per_sec = 0
Your thread_cache_size is fine
MAX CONNECTIONS
Current max_connections = 100
Current threads_connected = 2
Historic max_used_connections = 9
The number of used connections is 9% of the configured maximum.
You are using less than 10% of your configured max_connections.
Lowering max_connections could help to avoid an over-allocation of memory
See "MEMORY USAGE" section to make sure you are not over-allocating
INNODB STATUS
Current InnoDB index space = 3 M
Current InnoDB data space = 20 M
Current InnoDB buffer pool free = 0 %
Current innodb_buffer_pool_size = 8 M
Depending on how much space your innodb indexes take up it may be safe
to increase this value to up to 2 / 3 of total system memory
MEMORY USAGE
Max Memory Ever Allocated : 15.14 G
Configured Max Per-thread Buffers : 132.05 G
Configured Max Global Buffers : 3.25 G
Configured Max Memory Limit : 135.31 G
Physical Memory : 7.68 G
Max memory limit exceeds 90% of physical memory
KEY BUFFER
Current MyISAM index space = 16.92 G
Current key_buffer_size = 3.00 G
Key cache miss rate is 1 : 2326
Key buffer free ratio = 0 %
You could increase key_buffer_size
It is safe to raise this up to 1/4 of total system memory;
assuming this is a dedicated database server.
QUERY CACHE
Query cache is enabled
Current query_cache_size = 256 M
Current query_cache_used = 2 M
Current query_cache_limit = 256 M
Current Query cache Memory fill ratio = .94 %
Current query_cache_min_res_unit = 4 K
Your query_cache_size seems to be too high.
Perhaps you can use these resources elsewhere
MySQL won't cache query results that are larger than query_cache_limit in size
SORT OPERATIONS
Current sort_buffer_size = 256 M
Current read_rnd_buffer_size = 64 M
Sort buffer seems to be fine
JOINS
Current join_buffer_size = 8.00 M
You have had 408 queries where a join could not use an index properly
You have had 5836 joins without keys that check for key usage after each row
join_buffer_size >= 4 M
This is not advised
You should enable "log-queries-not-using-indexes"
Then look for non indexed joins in the slow query log.
OPEN FILES LIMIT
Current open_files_limit = 2158 files
The open_files_limit should typically be set to at least 2x-3x
that of table_cache if you have heavy MyISAM usage.
Your open_files_limit value seems to be fine
TABLE CACHE
Current table_open_cache = 1024 tables
Current table_definition_cache = 256 tables
You have a total of 119 tables
You have 175 open tables.
The table_cache value seems to be fine
TEMP TABLES
Current max_heap_table_size = 64 M
Current tmp_table_size = 1.00 G
Of 209033 temp tables, 7% were created on disk
Effective in-memory tmp_table_size is limited to max_heap_table_size.
Created disk tmp tables ratio seems fine
TABLE SCANS
Current read_buffer_size = 1.00 G
Current table scan ratio = 35358 : 1
read_buffer_size is over 8 MB there is probably no need for such a large read_buffer
TABLE LOCKING
Current Lock Wait ratio = 1 : 61064
Your table locking seems to be fine
存储过程已附加
BEGIN
create temporary table if not exists temp_cns (id int primary key, countryname varchar(255));
insert into temp_cns select c1.id, c2.countryname from cctonumbers c1 left join countrycodes c2 on c1.countrycode_id = c2.id;
drop temporary table if exists temp_cause_code_id;
SET @temp_ids = CONCAT('create temporary table temp_cause_code_id select id from billing_datas_NEW use index(primary) where ', conditions_str);
PREPARE stmt FROM @temp_ids;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
drop temporary table if exists temp_cause_code_allfileds;
create temporary table temp_cause_code_allfileds select m_tbl.id,m_tbl.destination,(select dialcode from cctonumbers where id = m_tbl.destination) as dial_code,(select number from cctonumbers where id = m_tbl.destination_country) as country_value,m_tbl.destination_country,m_tbl.release_cause_protocol_stack, m_tbl.binary_value_protocol_stack, m_tbl.release_cause,m_tbl.call_duration_seconds, m_tbl.destination_country as destination_name,m_tbl.start_time_date,m_tbl.start_time_date as start_date,m_tbl.carrierid_customer,m_tbl.technical_profileid_customer,m_tbl.origination_trunkid,m_tbl.carrierid_supplier,m_tbl.technical_profileid_supplier,m_tbl.supplierid,cm.carriername as customer,ctech.profilename as customer_tech,(select carriername from carriermasters where id = carrierid_supplier) as vendor,(select profilename from technicalprofiles where id = technical_profileid_supplier) as vendor_tech_profile from temp_cause_code_id as temp_ids LEFT JOIN billing_datas_NEW as m_tbl ON temp_ids.id = m_tbl.id LEFT JOIN carriermasters as cm ON m_tbl.carrierid_customer = cm.id LEFT JOIN technicalprofiles as ctech ON m_tbl.technical_profileid_customer = ctech.id WHERE m_tbl.id IS NOT NULL;
drop temporary table if exists temp_group_records;
SET @sqlv = CONCAT('create temporary table temp_group_records select destination,dial_code,country_value, start_date, destination_country, release_cause_protocol_stack, binary_value_protocol_stack, release_cause, destination_name, count(*) as calls, (sum(call_duration_seconds) / 60) as durations, sum(call_duration_seconds > 0) as scalls,customer,customer_tech,vendor,vendor_tech_profile from temp_cause_code_allfileds group by binary_value_protocol_stack, release_cause_protocol_stack, destination, destination_country ');
PREPARE stmt FROM @sqlv;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
update temp_group_records set destination = destination_country where destination = '';
update temp_group_records t, temp_cns c set t.destination_name = c.countryname where t.destination = c.id;
drop table if exists temp_records;
create table temp_records select customer,customer_tech,vendor,vendor_tech_profile,release_cause_protocol_stack, binary_value_protocol_stack,dial_code,country_value, release_cause, destination_name, sum(calls) as ncalls, sum(durations) as ndurations, sum(scalls) as nscalls,start_date from temp_group_records group by binary_value_protocol_stack, release_cause_protocol_stack, destination_name;
END
上述存储过程中的condition_str及其在mysql中的解释位于以下链接中
http://www.prosesindia.com/dock.docx
问题是
当我运行上述存储过程时，大约需要1分40秒。
有什么方法可以调整存储过程或服务器变量。
答案 0 :(得分：2)
首先，我真的不明白为什么要对可操作的数据源进行报告。您是否考虑过从现有结构中创建数据仓库？这些通常以支持您要运行的特殊查询的方式建模。
我真的没有理由以您使用它的方式通过存储过程进行报告。看看像Talend Open Studio这样的ETL工具，你可以在之前处理数据，然后将它加载到数据库中。
如果你想继续你现在的方式，我首先建议看一下填充表的查询* temp_cause_code_allfileds *你在那里使用了很多子选项，这些子选择在每条记录上进行评估基础。我想通过查询每个表的一般LEFT OUTER JOIN来做这件事会更有意义。
答案 1 :(得分：1)
在盯着它看了一段时间后，我可以说没有太多可以让它跑得更快。
关于服务器调优，您可以确保临时驱动器位于单独的磁盘而不是分区上，并且空间充足。
如果您的表中包含经常添加和删除记录的表，您还可以运行每周或每月表优化。
此外，如果您有可以废弃的旧数据，最好将其取出。
最后，我建议在所有表格和字段周围添加反斜杠，在结尾添加分号，大写所有命令。我已经看到这有助于减少少量的查询时间。但这对程序代码没有任何影响，只是来自编程语言的查询。