我正在编写节点js应用程序,它从SQL服务器数据库获取10万条记录并将其插入到mysql数据库中。我已完成代码但我的应用程序没有给我预期的性能。
我选择的100k记录需要9-10秒,批量插入过程需要3-4秒。我希望节点加速选择过程(从sql server选择记录需要3-4秒)。我试过使用2种方法,但仍然无法达到预期的时间。
/*First Way*/
var sql = require('mssql');
var mysql = require("mysql");
var sizeof = require('sizeof');
var moment = require('moment');
var config = {
user: '**',
password: '*****',
server: '******',
database: '*******'
}
var connection1 = mysql.createConnection({
host : 'localhost',
user : 'root',
password : 'root',
database : 'test'
});
connection1.connect();
sql.connect(config, function(err) {
if(err)
console.log(err);
console.log('Connecting to Sql Server');
console.time('Overall Time-Taken');
var request = new sql.Request();
console.time('Time-Taken to fetch');
request.query('select TOP 100000 * from ShipmentAuditLog WITH (NOLOCK)', function(err, recordset) {
if(err)
console.log(err);
else{
//console.log(sizeof(recordset));
console.timeEnd('Time-Taken to fetch');
syncing(recordset);
}
});
});
function mapping(row) {
var formatedString = '';
var CreatedDate = new moment(row.CreatedDate).format('YYYY-MM-DD HH:mm:ss');
row.Process = row.Process.replace(/'/g,"\\'");
if(row.Comment != null){
formatedString = row.Comment;
formatedString = formatedString.replace(/'/g,"\\'");
row.Comment = formatedString;
}
return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "', NOW())";
//return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "')";
}
var syncing = function(recordset){
//var columns = [];
var values = [];
/*for(col in recordset[0]){
columns.push(col);
}*/
/*for(var i=0,len = recordset.length;i<len;i+=1){
values.push(mapping(recordset[i]));
}*/
recordset.forEach(function(record){
values.push(mapping(record));
})
//console.log(values);
console.time('Time-Taken to Insert');
var QRY = connection1.query('INSERT INTO shipmentauditlog VALUES'+ values.join(','), function(err, result) {
if (err) {
console.log(err);
}
else {
console.timeEnd('Time-Taken to Insert');
console.timeEnd('Overall Time-Taken');
}
});
//console.log(QRY.sql);
}
/*Second Way*/
var sql = require('mssql');
var mysql = require("mysql");
var moment = require('moment');
var config = {
user: '**',
password: '***',
server: '****',
database: '****',
stream: true
}
var connection1 = mysql.createConnection({
host : 'localhost',
user : 'root',
password : 'root',
database : 'test'
});
connection1.connect();
var values = [];
function mapResult(row){
var formatedString = '';
var CreatedDate = new moment(row.CreatedDate).format('YYYY-MM-DD HH:mm:ss');
row.Process = row.Process.replace(/'/g,"\\'");
if(row.Comment != null){
formatedString = row.Comment;
formatedString = formatedString.replace(/'/g,"\\'");
row.Comment = formatedString;
}
return "('" + row.ShippingID + "','" + row.BagNo + "','" + row.ProcessLocation + "','" + row.Process + "','" + row.Comment + "','" + CreatedDate + "','" + row.CreatedBy + "','" + row.LastModifiedDate + "','" + row.LastModifiedBy + "','" + row.DestinationLocation + "','" + row.VenderLostShipmentsDebitId + "', NOW())";
}
var connection = new sql.Connection(config, function(err) {
if(err)
console.log(err);
console.log('Connecting to Sql Server');
console.time('Overall Time-Taken');
var request = new sql.Request(connection);
//request.stream = true;
request.query('select TOP 100000 * FROM ShipmentAuditLog WITH (NOLOCK)'); // or request.execute(procedure);
request.on('recordset',function(col){
console.time('Time-Taken to fetch');
});
// Emitted for each row
request.on('row', function(row) {
//Build array with resultset for bulk insert
values.push(mapResult(row));
});
request.on('error', function(err) {
console.log(err);
});
// Emitted for the last one
request.on('done', function(returnValue) {
console.timeEnd('Time-Taken to fetch');
// Function which perform bulk insert into mysql
syncing(values);
connection.close();
});
});
var syncing = function(values){
/*var startTime = new Date();
console.log("Start Time :"+ startTime);*/
console.time('Time-Taken to Insert');
var qry = connection1.query('INSERT INTO shipmentauditlog VALUES'+values.join(','),function(err,res){
if(err)
console.log(err);
else{
/*var endTime = new Date();
console.log('End Time:'+ endTime);*/
console.timeEnd('Time-Taken to Insert');
console.timeEnd('Overall Time-Taken');
}
});
}