遍历大型数据集的最佳方法

时间:2019-06-27 20:18:09

标签: javascript performance

我有一个很大的JSON值数据集。以下是该JSON对象的小快照。

 {
     "response": [
     {
         "data": [
         {
             "value": 1,
             "minute": "2019-06-10 11:51",
             "action": "firstApp",
         },
         {
             "value": 10,
             "minute": "2019-06-10 11:51",
             "action": "secondApp",
         },
         {
             "value": 100,
             "minute": "2019-06-10 11:51",
             "action": "thirdApp",
         },
         {
             "value": 10,
             "minute": "2019-06-10 11:52",
             "action": "firstApp",
         },
         {
             "value": 20,
             "minute": "2019-06-10 11:52",
             "action": "secondApp",
         },
         {
             "value": 115,
             "minute": "2019-06-10 11:52",
             "action": "thirdApp",
         }, ]
     }]
 }

现在考虑数据数组中大约有800个这些项。我想做的是创建一个JSON对象,该对象具有每个timestmap的值,就像下面的

[
   {
       "timestamp" : "2019-06-10 11:51",
       "firstApp" : {
           "value" : 1,
       }, 
       "secondApp": {
           "value":10,
       },
       "thirdApp": {
           "value" : 100,
       }
   },
   {
       "timestamp" : "2019-06-10 11:52",
       "firstApp" : {
           "value" : 10
       }, 
       "secondApp": {
           "value":20,
       },
       "thirdApp": {
           "value" : 115,
       }
   }
]

我已经这样编写了以下代码,但是它花费了很多时间(大约10-12秒)(这是很正常的)  请参考此CodeSandbox(https://codesandbox.io/s/objective-bartik-0r30j?autoresize=1&expanddevtools=1&fontsize=14&hidenavigation=1&module=%2Fsrc%2Findex.js)链接以查看实际代码。

我面临的问题是花很多时间(理应如此)

  1. 数据集很大
  2. 我的代码很糟糕-因为这里到处都有太多的循环

我无法控制1,但是我绝对可以控制2。您能否给我一些有关如何解决此问题的想法?

更新 这是“效果”标签中的屏幕截图 [![在此处输入图片描述] [1]] [1]

谢谢

let data = {
  response: [
    {
      data: [
        {
          value: 1,
          minute: "2019-06-10 11:51",
          action: "firstApp"
        },
        {
          value: 10,
          minute: "2019-06-10 11:51",
          action: "secondApp"
        },
        {
          value: 100,
          minute: "2019-06-10 11:51",
          action: "thirdApp"
        },
        {
          value: 10,
          minute: "2019-06-10 11:52",
          action: "firstApp"
        },
        {
          value: 20,
          minute: "2019-06-10 11:52",
          action: "secondApp"
        },
        {
          value: 115,
          minute: "2019-06-10 11:52",
          action: "thirdApp"
        }
      ]
    }
  ]
};

function massageData(data) {
  let historyData = [];
  let uniqueTimeStamps = [];
  let event = {
    timestamp: "",
    firstApp: {
      value: 0
    },
    secondApp: {
      value: 0
    },
    thirdApp: {
      value: 0
    }
  };

  for (var i = 0; i < data.length; i++) {
    let item = data[i];

    if (item.minute) {
      if (!uniqueTimeStamps.includes(item.minute)) {
        let timestamp = item.minute;
        console.log("--------------------");
        console.log(timestamp);
        event.timestamp = timestamp;
        event.firstApp.value = getDataValue(data, timestamp, "firstApp");
        event.secondApp.value = getDataValue(data, timestamp, "secondApp");
        event.thirdApp.value = getDataValue(data, timestamp, "thirdApp");
        console.log(event);
        historyData.push(event);
        uniqueTimeStamps.push(item.minute);
      }
    }
  }

  return historyData;
}

function getDataValue(data, timestamp, action) {
  for (var i = 0, len = data.length; i < len; i++) {
    let item = data[i];
    console.log(item);
    if (item["minute"] === timestamp && item["action"] === action) {
      return parseInt(item["value"]);
    }
  }
}

let workData = data.response[0].data;
let formattedData = massageData(workData);
console.log(formattedData);

2 个答案:

答案 0 :(得分:2)

您可以将经典哈希表用于相同的时间戳,并使用哈希表更新值。

function massageData(data) {
    var historyData = [],
        hash = Object.create(null),
        i, item;

    for (i = 0; i < data.length; i++) {
        item = data[i];
        if (!hash[item.minute]) {
            historyData.push(hash[item.minute] = { timestamp: item.minute, firstApp: { value: 0 }, secondApp: { value: 0 }, thirdApp: { value: 0 } });
        }
        hash[item.minute][item.action].value = item.value;
    }
    return historyData;
}

var data = { response: [{ data: [{ value: 1, minute: "2019-06-10 11:51", action: "firstApp" }, { value: 10, minute: "2019-06-10 11:51", action: "secondApp" }, { value: 100, minute: "2019-06-10 11:51", action: "thirdApp" }, { value: 10, minute: "2019-06-10 11:52", action: "firstApp" }, { value: 20, minute: "2019-06-10 11:52", action: "secondApp" }, { value: 115, minute: "2019-06-10 11:52", action: "thirdApp" }] }] },
    workData = data.response[0].data,
    formattedData = massageData(workData);

console.log(formattedData);
.as-console-wrapper { max-height: 100% !important; top: 0; }

答案 1 :(得分:0)

下面涉及将对象用作哈希表的另一种类似方法。它应该更容易理解:

function massageData (data) {
  const events = [];
  const timestamps = {};

  for (let i = 0; i < data.length; i++) {
    const { action, minute, ...item } = data[i];

    if (!(minute in timestamps)) {
      const event = { timestamp: minute };

      events.push(event);
      timestamps[minute] = event;
    }

    timestamps[minute][action] = item;
  }

  return events;
}

const data = [{ value: 1, minute: "2019-06-10 11:51", action: "firstApp" }, { value: 10, minute: "2019-06-10 11:51", action: "secondApp" }, { value: 100, minute: "2019-06-10 11:51", action: "thirdApp" }, { value: 10, minute: "2019-06-10 11:52", action: "firstApp" }, { value: 20, minute: "2019-06-10 11:52", action: "secondApp" }, { value: 115, minute: "2019-06-10 11:52", action: "thirdApp" }];

console.log(massageData(data));

这会将除minuteaction之外的每个项目中的所有属性复制到每个event中的对象中,作为计算后的属性[action]。如果value确实是每个item上唯一存在的其他属性,则可以简化代码:

function massageData (data) {
  const events = [];
  const timestamps = {};

  for (let i = 0; i < data.length; i++) {
    const { action, minute, value } = data[i];

    if (!(minute in timestamps)) {
      const event = { timestamp: minute };

      events.push(event);
      timestamps[minute] = event;
    }

    timestamps[minute][action] = { value };
  }

  return events;
}

const data = [{ value: 1, minute: "2019-06-10 11:51", action: "firstApp" }, { value: 10, minute: "2019-06-10 11:51", action: "secondApp" }, { value: 100, minute: "2019-06-10 11:51", action: "thirdApp" }, { value: 10, minute: "2019-06-10 11:52", action: "firstApp" }, { value: 20, minute: "2019-06-10 11:52", action: "secondApp" }, { value: 115, minute: "2019-06-10 11:52", action: "thirdApp" }];

console.log(massageData(data));