如何在dynamodb

Question

我正在使用带有nodejs的dynamodb，我有3000条记录，我在代码中写了60多个段，每个段扫描1mb的数据并将结果显示到60多个1 mb限制的段。因此，请提供解决方案如何在一个步骤中单步执行3000条记录扫描。请快速提供解决方案，因为我在项目中间遇到了困难。请帮我。以下是我的代码。

var AWS = require("aws-sdk");
var async = require("async");

    AWS.config.update({
      region: "---",
      endpoint: "-----------",
      accessKeyId: "-----------------",
      secretAccessKey:"----------"
    });


var db = new AWS.DynamoDB.DocumentClient()

var table = "rets_property_all";
var pstart =new Date () .getTime ();

async.parallel({
            0 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment:  0//by the worker who has been called
        },function (err , res) {
            callback (null , res.Items);
        });
            },
            1 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment: 1//by the worker who has been called
        }, function (err , res) {
            callback (null , res.Items);
        });
            },
      2 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment: 2//by the worker who has been called
        }, function (err , res) {
            callback (null , res.Items);
        });
            },
--------
---------
------

      62 : function(callback){
        db.scan ({TableName: table,
          ProjectionExpression: "#cityname,ListingKey ",
          FilterExpression: "#cityname = :v_id",
          ExpressionAttributeNames: {
              "#cityname": "CityName",
          },
          ExpressionAttributeValues: {":v_id" : 'BALTIMORE'},
            TotalSegments: 63,
            Segment: 62//by the worker who has been called
        }, function (err , res) {
            callback (null , res.Items);
        });
      },

        },function(err,results){
      if (err) {throw err; }
      var pend = new Date () .getTime ();

      console.log (results);
        })

Answer 1

实际上，无法覆盖扫描的 1 MB限制。这是DynamoDB设计限制，不能被任何API覆盖。您可能需要了解体系结构或AWS服务设计的限制。

您可以在后续扫描中使用LastEvaluatedKey从上一次扫描结束的位置开始。

  

扫描的结果集限制为每次调用1 MB。您可以使用   扫描响应中的LastEvaluatedKey可以检索更多结果。

用例不清楚为什么你想在一次扫描中获得所有3000条记录。即使如果您有特定的用例，也只能在DynamoDB扫描中实现。

甚至，在关系数据库中，您获取光标并完成迭代以按顺序获取所有记录。同样，在DynamoDB中，您必须递归使用Scan，直到LastEvaluatedKey为空。

递归进行单次扫描的示例代码，直到LastEvaluatedKey为空： -

var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
    TableName: table,
    ProjectionExpression: "#cityname,ListingKey ",
    FilterExpression: "#cityname = :v_id",
    ExpressionAttributeNames: {
        "#cityname": "CityName",
    },
    ExpressionAttributeValues: { ":v_id": 'BALTIMORE' }
};

docClient.scan(params, onScan);

function onScan(err, data) {
    if (err) {
        console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
    } else {
        // print all the movies
        console.log("Scan succeeded.");
        data.Items.forEach(function (movie) {
            console.log("Item :", JSON.stringify(movie));
        });

        // continue scanning if we have more movies
        if (typeof data.LastEvaluatedKey != "undefined") {
            console.log("Scanning for more...");
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            docClient.scan(params, onScan);
        }
    }
}