在完成数据库中的插入之前关闭连接

时间:2018-03-30 15:38:16

标签: node.js postgresql async-await

我正在Node.js中创建我的第一个应用程序,它应该按顺序执行这些操作:

  1. 连接到PostgreSQL数据库
  2. 创建一个表(如果它已存在,做任何事情)
  3. 对两个网站进行网页抓取并保存有关db
  4. 的信息
  5. 使用D3.js和数据库上的数据创建可视化
  6. 它与数据库断开连接。
  7. 我发布了有关插入操作的previous question。现在问题是async / await。

    我知道Node.js是异步的,而我应该同步执行操作。 我正在使用Node.js v8.9.4,所以我使用async / await构造。

    我读了一下'async / await如何,但我觉得很难正确使用它。

    这是我的应用程序的方案( app.js 文件):

    const postgreSQLlib = require('./middlewares/postgreSQLlib.js')
    const scraperCovIt = require('./routers/scraperCovIt.js');
    const scraperCov = require('./routers/scraperCov.js');
    
    const start = async function() {
        // STEP 0 - Start
        await console.log('\nSTART');
    
        // STEP 1 - Connect to db
        await postgreSQLlib.connect();
    
        // STEP 2 - Create tables
        var queryCreateCoverages = {
            text: 'CREATE TABLE IF NOT EXISTS coverages ('+
                        'id SERIAL PRIMARY KEY,' +
                        'vaccine VARCHAR(64) NOT NULL,' + 
                        'country VARCHAR(255) NOT NULL,' +
                        'region VARCHAR(255),' +
                        'year VARCHAR(4) NOT NULL,' +
                        'value VARCHAR(12) NOT NULL);'
        };
        var queryRes = await postgreSQLlib.query(queryCreateCoverages, '[CREATE TABLE coverages]');
    
        // STEP 3 - Get data
        await scraperCovIt.download();
        await scraperCov.download();
    
        // STEP 4 - Disconnect from db
        await postgreSQLlib.disconnect();
    
        // STEP 5 - End
        return '\nFINISH';
    }
    
    // start application
    start()
    .then(function(res) {
        console.log(res);
    })
    .catch(function(err) {
        console.log(err);
    });
    

    postgreSQLlib.js

    const {Client} = require('pg'); 
    const connectionString = 'postgres://admin:admin@localhost:5432/db';
    let client;
    var methods = {};
    
    methods.connect = async function() {
        client = new Client({connectionString});
        return await client.connect()
        .then(async function() {
            await console.log('\nConnected to ' + client.database + ' at ' + client.host + ':' + client.port + ' as ' + client.user + ' (pass: ' + client.password + ')');
        })
        .catch(function(err) {
            console.log('\nError during connection to PostgreSQL');
            throw err;
        });
    }
    
    methods.query = async function(query, print) {
        return await client.query(query)
        .then(function(res) {
            console.log(print, 'OK query');
            return res;
        })
        .catch(function(err) { 
            console.log(print, 'ERR query');
        });
    }
    
    methods.disconnect = async function() {
        return await client.end()
        .then(function() {
            console.log('\nConnection has ended');
        })
        .catch(function(err) {
            console.log('\nError during clossing connection');
            throw err;
        }); 
    }
    
    module.exports = methods;
    

    scraperCov.js

    var cheerio = require('cheerio');
    var request = require('request-promise');
    var postgreSQLlib = require('../middlewares/postgreSQLlib.js');
    
    var methods = {};
    
    var countries = {
        'Albania': 'ALB',
        'Austria': 'AUT',
        'Belgium': 'BEL'
    };
    
    methods.download = async function(req, res) {
        for(country in countries) {
            console.log('\nCOUNTRY:', country);
            var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];
            let res = await request(url);
            insert(res);
        }
    }
    
    module.exports = methods;
    
    let insert = async function(html) {
        $ = cheerio.load(html);
    
        var years = [];
        var vaccines = [];
        var coverages = [];
    
        $('.ts .year').each(function() {
            years.push($(this).text().trim());
        });
        $('.ts .odd td a, .ts .even td a').each(function() {
            vaccines.push($(this).text().trim());
        });
        $('.ts .odd .statistics_small, .ts .even .statistics_small').each(function() {
            coverages.push($(this).text().trim());
        });
    
        const numYears = years.length;
        const numVaccines = vaccines.length;
        for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
            for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
                let obj = {
                    year: years[yearIdx],
                    country: country,
                    region: '',
                    vaccine: vaccines[vaccineIdx],
                    coverage: coverages[vaccineIdx*numYears + yearIdx]
               }
    
                // save on db
                const queryInsert = {
                    text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5);',
                    values: [vaccines[vaccineIdx], country, '', years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]]
                }
                var printText = '[INSERT ' + country + ' IN coverages]';
                var queryRes = await postgreSQLlib.query(queryInsert, printText);
            }
        }
    }
    

    scraperCovIt.js

    var textract = require('textract');
    var postgreSQLlib = require('../middlewares/postgreSQLlib.js');
    
    var methods = {};
    var jsons = [];
    var mainUrl = 'http://www.salute.gov.it/portale/documentazione/p6_2_8_3_1.jsp?id=20';
    var urls = [ 
        {year: '2013', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_0_fileAllegati_itemFile_1_file.pdf'}, 
        {year: '2012', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_5_fileAllegati_itemFile_0_file.pdf'}, 
        {year: '2011', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_6_fileAllegati_itemFile_0_file.pdf'}
    ];
    
    methods.download = async function(req, res) {
        await extractText();
    }
    
    async function extractText() {
        var config = {
            preserveLineBreaks: true
        };
        urls.forEach(function(url) {
            textract.fromUrl(url.link, config, async function(error, text) {
                if(error) {
                    throw error;
                }
                switch(url.year) {
                    case '2011': 
                    case '2012':
                        await extractTextType1(url, text);
                        break;
                    case '2013': 
                        await extractTextType2(url, text);
                        break;
                    default:
                        await console.log('Error: no case');
                }
            });
        });
    }
    
    async function extractTextType1(url, text) {
        var matrix = [];
        var map = [];
        var vaccines = [];
        var regionsTemp = [];
        var regions = [];
        var regionLength = [1, 2, 1, 2, 3, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
    
        var textArray = text.split('\n');
        for(var i = 0; i < 23; i++) {
            matrix[i] = textArray[i].split(' ');
        }
    
        matrix[0].shift();
        vaccines = matrix[0];
        map[0] = vaccines;
    
        for(var i = 0; i < regionLength.length; i++) {
            var j = i + 1; 
            var indexToRemove = 0;
            var numberToRemove = regionLength[i];
            var region = matrix[j].splice(indexToRemove, numberToRemove);
            regionsTemp.push(region);
            map[j+1] = matrix[j];
        }
    
        for(var i = 0; i < regionsTemp.length; i++) {
            var region = '';
            if(regionLength[i] > 1) {
                region = regionsTemp[i].join(' ');
            }
            else {
                region = regionsTemp[i].join('');
            }
            regions.push(region);
        }
        map[1] = regions;
    
        for(var i = 0; i < map.length; i++) {
            for(var j = 0; j < map[i].length; j++) {
                map[i][j] = map[i][j].replace(/\r/g, '');
            }
        }
    
        vaccines = map.shift();
        regions = map.shift();
    
        var thisJson = map.reduce(function(result, v, i) {
            v.forEach(async function(o, k) {
                var obj = createJsonObjectCoverage(url.year, 'Italy', vaccines[k], regions[i], o);
                // save on db
                const queryInsertDoNothing = {
                    text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (vaccine, country, region, year, value) DO NOTHING;',
                    values: [vaccines[k], 'Italy', regions[i], url.year, o]
                };
                var printText = '[INSERT Italy IN coverages]';
                var queryRes = await postgreSQLlib.query(queryInsertDoNothing, printText);
            });
            return result;
        }, jsons);
    }
    
    async function extractTextType2(url, text) {
        // similar code to extractTextType1(url, text)
    }
    
    function createJsonObjectCoverage(year, country, vaccine, region, coverage) {
        return {
            year: year, 
            country: country, 
            region: region, 
            vaccine: vaccine, 
            coverage: coverage
        };
    }
    
    module.exports = methods;
    

    问题是数据库中的插入不能正常工作。

    使用 scraperCov.js 运行 app.js 的结果是:

    START
    
    Connected to db at localhost:5432 as admin (pass: admin)
    [CREATE TABLE coverages] OK query
    
    COUNTRY: Albania
    
    COUNTRY: Austria
    [INSERT Albania IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    
    COUNTRY: Belgium
    [INSERT Austria IN coverages] OK query
    [INSERT Austria IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    [INSERT Belgium IN coverages] OK query
    
    Connection has ended
    
    FINISH
    [INSERT Belgium IN coverages] ERR query
    

    从打印件中可以看出,最后会出现错误,因为在插入结束之前连接已关闭。 此外,插入不是有序的(在单词COUNTRY: Albania下,没有任何内容,而在单词COUNTRY: Austria下有一些与阿尔巴尼亚相关的查询的结果等。)。

    使用 scraperCovIt.js 运行 app.js 的结果是:

    START
    
    Connected to db at localhost:5432 as admin (pass: admin)
    [CREATE TABLE coverages] OK query
    
    Connection has ended
    
    FINISH
    

    在这种情况下,插入不会完全执行。

    我该如何解决?

    这些是我试图解决这个问题的日子,但我不知道该怎么做。这是我第一次使用async/await而且我遇到了很多麻烦。

    感谢任何想要帮助我的人

1 个答案:

答案 0 :(得分:3)

正如我们在评论中所讨论的那样,这里有很多错误。以下是部分概念列表:

    如果等待与异步操作完成直接相关的承诺,则
  1. await仅等待异步操作。在承诺以外的任何事情上使用await并没有做任何有用的事情。它没有神奇的力量来知道什么时候进行一些底层的异步操作。 await只遵循承诺。

  2. {li>

    await.forEach()循环内不会暂停该循环的执行。

  3. 不要混合回调代码和承诺代码。将它们混合起来真的很难让它们正常工作。如果你有一些只接受回调的代码,那么将它转换为一个返回promise的新函数。您通常可以使用util.promisify()在一行代码中创建新函数。

  4. 我不可能攻击你的整个代码,因为我甚至都不知道它是做什么的,但我会挑选一些例子并告诉你如何修复它们。我开始查看返回promise的extractText()(仅因为它声明为async,但该承诺根本没有连接到其中的异步操作。

  5. 这让我了解了extractText()调用的一些功能,例如extractTextType1(),因此我将首先向您展示如何解决此问题。

  6. extractTextType1()中,您在await postgreSQLlib.query(...)循环内的.forEach()循环内使用.reduce()await不会导致这些类型的循环中的任何一个暂停。因此,extractTextType1()函数将在完成任何查询之前很久就返回。由于看起来这些查询(实际上是插入的)不必排序并且可以并行运行,我们可以简单地将所有查询中的promise收集到一个数组中,然后使用Promise.all()来跟踪它们何时都完成了。

    该代码如下所示:

    function extractTextType1(url, text) {
        var matrix = [];
        var map = [];
        var vaccines = [];
        var regionsTemp = [];
        var regions = [];
        var regionLength = [1, 2, 1, 2, 3, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
    
        var textArray = text.split('\n');
        for(var i = 0; i < 23; i++) {
            matrix[i] = textArray[i].split(' ');
        }
    
        matrix[0].shift();
        vaccines = matrix[0];
        map[0] = vaccines;
    
        for(var i = 0; i < regionLength.length; i++) {
            var j = i + 1; 
            var indexToRemove = 0;
            var numberToRemove = regionLength[i];
            var region = matrix[j].splice(indexToRemove, numberToRemove);
            regionsTemp.push(region);
            map[j+1] = matrix[j];
        }
    
        for(var i = 0; i < regionsTemp.length; i++) {
            var region = '';
            if(regionLength[i] > 1) {
                region = regionsTemp[i].join(' ');
            }
            else {
                region = regionsTemp[i].join('');
            }
            regions.push(region);
        }
        map[1] = regions;
    
        for(var i = 0; i < map.length; i++) {
            for(var j = 0; j < map[i].length; j++) {
                map[i][j] = map[i][j].replace(/\r/g, '');
            }
        }
    
        vaccines = map.shift();
        regions = map.shift();
    
        let promises = [];
        var thisJson = map.reduce(function(result, v, i) {
            v.forEach(async function(o, k) {
                var obj = createJsonObjectCoverage(url.year, 'Italy', vaccines[k], regions[i], o);
                // save on db
                const queryInsertDoNothing = {
                    text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (vaccine, country, region, year, value) DO NOTHING;',
                    values: [vaccines[k], 'Italy', regions[i], url.year, o]
                };
                var printText = '[INSERT Italy IN coverages]';
                promises.push(postgreSQLlib.query(queryInsertDoNothing, printText));
            });
            return result;
        }, jsons);
        return Promise.all(promises);
    }
    

    现在extraTextType1()返回一个只有在内部完成所有异步操作后才能解析的promise。如果其中任何一个失败,它将拒绝。

    另请注意,extraTextType1()不再声明为async,因为没有使用await。它只是直接返回一个承诺。如果你需要对这些插入进行排序,那么循环需要更彻底的重写才能一次完成。

    假设您对extraTextType2()应用了类似的修补程序,现在让我们修复包含函数extractText()。在这里,我们宣传textract.fromUrl(),因此我们不会尝试混合回调和承诺,我们会将.forEach()循环更改为for/of循环,以便await可以使用循环。然后,从await

    中删除console.log()
    const util = require('util');
    const textractFromUrl = util.promisify(textract.fromUrl);
    
    async function extractText() {
        var config = {
            preserveLineBreaks: true
        };
        for (let url of urls) {
            let text = await textractFromUrl(url.link, config);
            switch(url.year) {
                case '2011': 
                case '2012':
                    await extractTextType1(url, text);
                    break;
                case '2013': 
                    await extractTextType2(url, text);
                    break;
                default:
                    console.log('Error: no case');
            }
        }
    }
    

    然后,您可以通过更改以下内容来清除下载功能:

    methods.download = async function(req, res) {
        await extractText();
    }
    

    到此:

    methods.download = function(req, res) {
        return extractText();
    }
    
相关问题