NodeJS，MSSQL Bulk从API获取并保存天气观测值

时间：2019-03-11 07:12:29

标签： node.js json sql-server

我正在尝试编写一个Nodejs文件，以循环浏览来自气象局JSON文件的700个天气预报。每个文件都包含特定气象站点在过去24小时内的许多天气观测值：air temp，gust_kmh等。对于每个文件，我想查找每个观测值的最小值/最大值/平均值（air_temp_min，air_temp_max，air_temp_avg ）。然后，我想将此提交到MSSQL数据库。我打算每天执行一次此操作，因此每个位置观测值将在数据库中以多行作为一行输入。

我不熟悉异步操作，并且在将一个操作/功能/承诺链接到下一个操作或功能或超过最大侦听器方面一直遇到麻烦。

因此，我需要一个优雅或有效的建议或指导。我不介意使用哪个包，我尝试过请求，axios，mssql，乏味。我可以让他们全部单独工作一行，但不能以一种和谐的方式在一起工作。

起点是遍历每个站点的URL。这是一个示例。

http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95767.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95772.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94650.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95774.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95754.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95747.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95779.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94774.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94781.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95770.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95758.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.99738.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94775.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94776.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94773.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95773.json,
http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94588.json

谢谢。

1 个答案:

答案 0 :(得分：1)

因此，您可以使用模块请求（+ request-promise-native），mssql，然后使用lodash（用于聚合，例如最大，最小，平均）和时刻进行日期解析。这是一些示例代码，可以满足您的要求。

您需要运行以下cmd行以安装依赖项：

npm install mssql request request-promise-native lodash moment --save

现在关于安排调用，我建议也许只是设置一个安排好的任务，每天调用一次脚本（取决于您的操作系统）。

一些待办事项..也许要确保我们很好地关闭连接等，并在某些情况下阻止某些异常处理！

运行此命令后，我在数据库中看到以下行：

天气数据 

timestamp               |location_name                  |air_temp_max |air_temp_min |air_temp_avg
-----------------------------------------------------------------------------------------------------
2019-03-11 00:00:00.000 |Lake Macquarie - Cooranbong    |32.4         |17.8         |23.1027777777778
2019-03-11 00:00:00.000 |Maitland Airport               |33.9         |19.9         |24.4478260869565
2019-03-11 00:00:00.000 |Mangrove Mountain              |30           |18.1         |22.3784722222222
2019-03-11 00:00:00.000 |Merriwa                        |34.8         |18.6         |25.4435897435898
2019-03-11 00:00:00.000 |Murrurundi Gap                 |31.4         |17.4         |23.3627906976744
2019-03-11 00:00:00.000 |Nelson Bay                     |26.4         |22.5         |24.7
2019-03-11 00:00:00.000 |Newcastle Nobbys               |27.7         |21.1         |23.1243055555556
2019-03-11 00:00:00.000 |Newcastle University           |26.8         |21.6         |24.2
2019-03-11 00:00:00.000 |Norah Head                     |29.5         |20.6         |23.3048611111111
2019-03-11 00:00:00.000 |Scone Airport                  |35.8         |20.4         |26.0236363636363
2019-03-11 00:00:00.000 |Singleton (Defence)            |34.5         |20           |25.1141891891892
2019-03-11 00:00:00.000 |Tocal                          |33.6         |20           |24.7993055555556
2019-03-11 00:00:00.000 |Williamtown                    |31.5         |20.7         |23.6551724137931
2019-03-11 00:00:00.000 |Armidale                       |20.7         |19.8         |20.3
2019-03-11 00:00:00.000 |Armidale Airport               |29.7         |13.9         |22.0801282051282
2019-03-11 00:00:00.000 |Glen Innes Airport             |29.7         |15.1         |21.8228187919463

天气模式SQL 

CREATE TABLE weather_data (
    timestamp datetime,
    location_name nvarchar(200),
    air_temp_max float,
    air_temp_min float,
    air_temp_avg float,
);

节点代码：

index.js 

const rp = require('request-promise-native');
const _ = require('lodash');
const sql = require('mssql');
const moment = require('moment');

const config = {
    user: 'db_user',  // Change as appropriate
    password: 'pass',  // Change as appropriate
    server: 'localhost', // Change as appropriate
    database: 'Weather',
    port: 1433 // Default port, change as appropriate
}

let sqlConnPool = null;

const urlList = [
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95767.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95772.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94650.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95774.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95754.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95747.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95779.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94774.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94781.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95770.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95758.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.99738.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94775.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94776.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94773.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.95773.json',
    'http://reg.bom.gov.au/fwo/IDN60801/IDN60801.94588.json'
];

async function downloadStationData() {
    for(url of urlList)  {
        let res = await downloadAndSaveWeatherData(url);
        console.log("Res: ", res);
    }
}

sql.connect(config).then(pool => {
    console.log("Connected to database...");
    sqlConnPool = pool;
    downloadStationData()
});

async function downloadAndSaveWeatherData(url) {
    console.log("downloadAndSaveWeatherData: Downloading from: ", url);
    let weatherData = await downloadWeatherData(url);
    if (weatherData.observations && weatherData.observations.data && weatherData.observations.data.length > 0) {
        console.log("downloadAndSaveWeatherData: Observations (count): ", weatherData.observations.data.length);
        let processedData = processWeatherData(weatherData.observations.data);
        console.log("downloadAndSaveWeatherData: processed data: ", processedData);
        let writeResult = await writeWeatherDataToDatabase(processedData);
        return { status: "OK"};
    }
}

// Get max, min, average etc.
function processWeatherData(data) {
    const result = { 
        timestamp: moment(data[0].aifstime_utc.substring(0,8)).toDate(),
        location_name: data[0].name 
    };

    const fields = ["air_temp"];

    fields.forEach(field => {
        result[`${field}_max`] = _.max(data.map(row => row[field])),
        result[`${field}_min`] = _.min(data.map(row => row[field])),
        result[`${field}_avg`] = _.mean(data.map(row => row[field]))
    });

    return result;
}

function downloadWeatherData(url) {
    const options = { url: url, json: true, resolveWithFullResponse: true }
    return rp(options).then((response) => response.body);
}

function writeWeatherDataToDatabase(processedData) {
    const request = sqlConnPool.request()
    return new Promise((resolve, reject) => {
        request.input('timestamp', sql.DateTime, processedData.timestamp);
        request.input('location_name', sql.NVarChar, processedData.location_name);
        request.input('air_temp_max', sql.Float, processedData.air_temp_max);
        request.input('air_temp_min', sql.Float, processedData.air_temp_min);
        request.input('air_temp_avg', sql.Float, processedData.air_temp_avg);
        request.query('insert into weather_data (timestamp, location_name, air_temp_max, air_temp_min, air_temp_avg) values (@timestamp, @location_name, @air_temp_max, @air_temp_min, @air_temp_avg)',
        (err, result) => {
            if (err) {
                console.error("writeWeatherDataToDatabase: Error occurred: ", err);
                reject(err);
            } else {
                resolve(result);
            }
        });
    });
}
相关问题
最新问题