Question

我们的MySQL数据库中有一些大表。列/行的数量不是很大，但是每行中都包含JSON。现在，我们正在尝试从MySQL读取数据并将其完全存储为JSON文件，但是该过程正在消耗超过3GB的内存。我们尝试了以下方法：

async function cacheItems() {
  let [objs] = await db.query('SELECT o.obj_id FROM objs o;');
  for (var i in objs) {
    console.log('OBJ ' + objs[i].obj_id);
    let [rows] = await db.query('SELECT JSON_OBJECTAGG(i.title, i.details) AS result FROM items_json i WHERE i.obj_id=? AND i.details IS NOT NULL GROUP BY i.obj_id;', [objs[i].obj_id]);
    if (!rows.length) {
      continue;
    }
    fs.outputFileSync(path.join(__dirname, 'cache', 'items', objs[i].obj_id + '.json'), JSON.stringify({ data: rows[0].result, message: 'Success' }));
    delete rows;
  }
  delete objs;
}

async function cacheItems2() {
  let [objs] = await db.query('SELECT o.obj_id FROM objs o;');
  // Items
  for (var i in objs) {
    console.log('OBJ ' + objs[i].obj_id);
    var [rows] = await db.query('SELECT i.title, i.details, i.details_updated AS updated FROM items_json i WHERE i.obj_id=? AND i.details IS NOT NULL ORDER BY i.title ASC;', [objs[i].obj_id]);
    rows = rows.reduce(function (map, obj) {
      map[obj.title] = obj.details;
      return map;
    }, {});
    fs.outputFileSync(path.join(__dirname, 'cache', 'items', objs[i].obj_id + '.json'), JSON.stringify({ data: rows, status: message: 'Success' }));
    delete rows;
  }
  delete objs;
}

// Here with disabled "typeCast" so JSON will not be converted to Object.
async function cacheItems3() {
  let [objs] = await db.query('SELECT o.obj_id FROM objs o;');
  for (var i in objs) {
    console.log('OBJ ' + objs[i].obj_id);
    let [rows] = await db.query('SELECT JSON_OBJECTAGG(i.title, i.details) AS result FROM items_json i WHERE i.obj_id=? AND i.details IS NOT NULL GROUP BY i.obj_id;', [objs[i].obj_id]);
    if (!rows.length) {
      continue;
    }
    fs.outputFileSync(path.join(__dirname, 'cache', 'items', objs[i].obj_id + '.json'), '{"data": ' + rows[0].result + ', "message": "Success" }');
    delete rows;
  }
  delete objs;
}

除了内存消耗和有时使整个过程崩溃之外，它们两者都运行良好。

我当时正在考虑使用Streams，但是问题是我不仅需要将结果存储在JSON文件中，而且还需要将其包装在另一个JSON对象中：

{ data: RESULT_HERE, message: 'Success' }

是否存在一些降低内存消耗并获得结果的好方法？非常感谢！

使用MySQL的NodeJS编写大型JSON文件

0 个答案: