mysql万行express.js响应慢

时间:2015-10-28 21:14:38

标签: mysql node.js csv express

我正在使用node.js v0.10.33和node-mysql 2.5.4以及express.js 4.10.2和fast-csv 0.6.0

在请求“查询9000000行并将其下载为csv”后,对服务器发出的任何后续请求都会变慢。有没有更好的方法来避免缓慢的反应?

/**
 * problem is why the server response slows down on a large request mande to mysqlrows_to_csv which queries 9000000 rows and saves as csv
 */

(function () {
    'use strict';
    var async = require('async');
    var csv = require("fast-csv");
    var fs = require('fs');
    var mysql = require('mysql');
    //var express = require('express');
    //var expRouter = express.Router();   // Express Router

    expRouter.get('/mysqlrows_to_csv/', function(req, res, next) {
        mysqlrows_to_csv(function(err) {
            console.log('use socket to emit a notification to client saying csvfile is ready');
        });
        res.json({'mysqlrows_to_csv': true});
        return res.end();
    });

    expRouter.get('/some_other_queries/', function(req, res, next) {
        res.json({'some_other_queries': true});
        return res.end();
    });


    function mysqlrows_to_csv(callback) {
        var csvStream;
        var connProp = {
                host     : 'host',
                user     : 'user',
                password : 'password',
                database : 'database'
        };
        var I = 0;
        async.waterfall([
            function(cb) {
                var connection = mysql.createConnection(connProp);
                connection.connect(function(err) {
                    if (err) return callback(err);
                    csvStream = csv.createWriteStream({headers: true});
                    var writableStream = fs.createWriteStream("dump_mysql_to_csv.csv");
                    writableStream.on("error", function(err){
                        return callback(err);
                    });
                    writableStream.on("finish", function(){
                        console.log("DONE!");
                        return callback(null);
                    });
                    if (csvStream) csvStream.pipe(writableStream);
                    return cb(err, connection);
                });
            },
            function(connection, cb) {
                var qry = 'select * from table';    // this has aleast 9000000 rows
                var query = connection.query(qry);
                query
                    .on('error', function(err){
                        return cb(err);
                    })
                    .on('result', function(row) {
                        connection.pause();
                        I = I + 1;
                        console.log('row no.', I);
                        if (csvStream) csvStream.write(row);
                        connection.resume();
                    })
                    .on('end', function() {
                        console.log('stream-ended')
                        if (csvStream) csvStream.end();
                        return cb(null, connection);
                    });
            },
            function(connection, cb) {
                connection.end(function(err) {
                    if (err) {
                        console.log('Error while disconnecting mysql', err);
                    }
                    console.log('The connection is terminated now');
                    console.log('final conn state:', connection.state);
                });
            }
        ]);
    }
}());

0 个答案:

没有答案