如何使用node.js从`AWS dynamodb`获取/扫描所有项目

时间：2017-06-16 13:03:51

标签： node.js amazon-dynamodb aws-lambda

如何使用AWS dynamodb从node.js获取/扫描所有项目。我在这里发布我的代码。

var docClient = new aws.DynamoDB.DocumentClient();
    var params = {
    TableName:"users",
    KeyConditionExpression:"user_status=:status",
    ExpressionAttributeValues: {
        ":status": "Y"
    }
    };

    var queryExecute = function(callback) {
        docClient.query(params,function(err,result) {
            if(err) {
                console.log(err)
                callback(err);
                } else {
                console.log(result);

                if(result.LastEvaluatedKey) {
                    params.ExclusiveStartKey = result.LastEvaluatedKey;
                    queryExecute(callback);
                    } else {
                        callback(err,items);
                    }
                }
            });
        }
        queryExecute(callback);

这让我失误。

ValidationException: Query condition missed key schema element: `user_id`.

这里的主键是user_id。我不想在查询条件中使用它，因为如果我在KeyConditionExpression中提到主键，我需要设置一个值。可能是我错了。不过请建议我从dynamodb获取user_status = "Y"的所有项目的好方法

12 个答案:

答案 0 :(得分：22)

如果您想在不使用Hash密钥值的情况下从DynamoDB获取数据，则需要使用Scan API。

注意： Scan API会读取表格中的所有项目以获取结果。因此，在DynamoDB中这是一项代价高昂的操作。

替代方法：使用GSI

上述情景的扫描代码： - 

var docClient = new AWS.DynamoDB.DocumentClient();

var params = {
    TableName: "users",
    FilterExpression: "#user_status = :user_status_val",
    ExpressionAttributeNames: {
        "#user_status": "user_status",
    },
    ExpressionAttributeValues: { ":user_status_val": 'somestatus' }

};

docClient.scan(params, onScan);
var count = 0;

function onScan(err, data) {
    if (err) {
        console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
    } else {        
        console.log("Scan succeeded.");
        data.Items.forEach(function(itemdata) {
           console.log("Item :", ++count,JSON.stringify(itemdata));
        });

        // continue scanning if we have more items
        if (typeof data.LastEvaluatedKey != "undefined") {
            console.log("Scanning for more...");
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            docClient.scan(params, onScan);
        }
    }
}

答案 1 :(得分：6)

这对我有用：

export const scanTable = async (tableName) => {
    const params = {
        TableName: tableName,
    };

    let scanResults = [];
    let items;
    do{
        items =  await documentClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey  = items.LastEvaluatedKey;
    }while(typeof items.LastEvaluatedKey != "undefined");

    return scanResults;

};

答案 2 :(得分：2)

AWS文档示例不适用于我。 @Hank方法可以解决问题。

在lambda中使用处理程序：

const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
    // optional tuning - 50% faster(cold) / 20% faster(hot)
    apiVersion: '2012-08-10',
    sslEnabled: false,
    paramValidation: false,
    convertResponseTypes: false
});

const tableName = 'series';

exports.handler = async (event, context, callback) => {
    let params = { TableName: tableName };

    let scanResults = [];
    let items;

    do {
        items = await docClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey = items.LastEvaluatedKey;
    } while (typeof items.LastEvaluatedKey != "undefined");

    callback(null, scanResults);
};

答案 3 :(得分：2)

我使用这样的承诺：

let AWS = require('aws-sdk');
let docClient = new AWS.DynamoDB.DocumentClient();

async function dbRead(params) {
    let promise = docClient.scan(params).promise();
    let result = await promise;
    let data = result.Items;
    if (result.LastEvaluatedKey) {
        params.ExclusiveStartKey = result.LastEvaluatedKey;
        data = data.concat(await dbRead(params));
    }
    return data;
}

并使用它：

let params = {
  TableName: 'Table'
};
let data = await dbRead(params);

答案 4 :(得分：1)

一个节点表示解决方案，它以JSON格式返回数据：

let datapack=[];
item = {
        TableName: ddbTable,
        FilterExpression: "aws = :e AND begins_with ( Id, :t )",
        ExpressionAttributeValues: {
            ":t"    :   "contact",
            ":e"    :   aws
        },
        ProjectionExpression: "Id,FirstName,LastName,cEmail",
    };
    docClient.scan(item, onScan);
    function onScan(err, data) {
        if (err) {
            console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
        } else {        
            datapack = datapack.concat(data.Items);
            });
            if (typeof data.LastEvaluatedKey != "undefined") {
                item.ExclusiveStartKey = data.LastEvaluatedKey;
                docClient.scan(item, onScan);
            } else {
                res.json(datapack);
            }
        }
    }

答案 5 :(得分：1)

对于不使用AWS.DynamoDB.DocumentClient的用户，此解决方案将起作用。 为了方便阅读和使用异步/等待，我将功能分为多个模块。

const AWS = require("aws-sdk");
AWS.config.update({
    // update table region here
    region: "us-west-2"
});
var dynamodb = new AWS.DynamoDB();
const performAsynScanOperation = (scanParams) => {
    return new Promise((resolve, reject) => {
        dynamodb.scan(scanParams, function (err, responseData) {
            if (err) {
                reject(err)
            } else {
                resolve(responseData)
            }
        })
    })
}

const getAllRecords = async (tableName) => {
    let allItems = [];
    let LastEvaluatedKeyFlag = true;
    let scanParams = { TableName: tableName }
    while (LastEvaluatedKeyFlag) {
        let responseData = await performAsynScanOperation(scanParams)
        let batchItems = responseData.Items;
        allItems = allItems.concat(batchItems);
        if (responseData.LastEvaluatedKey) {
            LastEvaluatedKeyFlag = true;
            console.log('LastEvaluatedKey', responseData.LastEvaluatedKey)
            scanParams.ExclusiveStartKey = responseData.LastEvaluatedKey
        } else {
            LastEvaluatedKeyFlag = false;
        }
    }
    return allItems;
}
getAllRecords('<Name of table>').then((allItems)=>{
  console.log(allItems)
})

答案 6 :(得分：0)

这里是一个答案，它将以JSON格式node / express提供索引结果，而不是使用昂贵的扫描。请注意docClient.query的使用：

 datapack=[];
    item = {
        TableName: ddbTable,
        IndexName: "cEmailIndex",
        KeyConditionExpression : "aws = :e AND begins_with ( cEmail, :t )",
        ExpressionAttributeValues: {
            ":t"    :   search,
            ":e"    :   aws
        },
        ProjectionExpression: "Id,FirstName,LastName,cEmail",
    };
    docClient.query(item, onScan);
    function onScan(err, data) {
        if (err) {
            console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
        } else {
            datapack = datapack.concat(data.Items);
            if (typeof data.LastEvaluatedKey != "undefined") {
                item.ExclusiveStartKey = data.LastEvaluatedKey;
                docClient.query(item, onScan);
            } else {
                // console.log(JSON.stringify(datapack));
                res.json(datapack);
            }
        }
    }

答案 7 :(得分：0)

使用Promise and async 

const aws = require('aws-sdk');
aws.config.update({ region: 'us-east-1' });
const documentClient = new aws.DynamoDB.DocumentClient();

const scanAll = async (params) => {
  let lastEvaluatedKey = 'dummy'; // string must not be empty
  const itemsAll = [];
  while (lastEvaluatedKey) {
    const data = await documentClient.scan(params).promise();
    itemsAll.push(...data.Items);
    lastEvaluatedKey = data.LastEvaluatedKey;
    if (lastEvaluatedKey) {
      params.ExclusiveStartKey = lastEvaluatedKey;
    }
  }
  return itemsAll;
}

像这样使用

const itemsAll = scanAll(params);

查询代码相同（只需将扫描替换为查询）

答案 8 :(得分：0)

您可以从@aws/dynamodb-query-iterator使用ScanPaginator：

import { ScanPaginator } from '@aws/dynamodb-query-iterator';
import DynamoDB = require('aws-sdk/clients/dynamodb');

const paginator = new ScanPaginator(
  new DynamoDB.DocumentClient(),
  {
    TableName: "users",
    FilterExpression: "#user_status = :user_status_val",
    ExpressionAttributeNames: {
      "#user_status": "user_status",
    },
    ExpressionAttributeValues: { ":user_status_val": 'somestatus' }
  }
);

for await (const page of paginator) {
    // do something with `page`, e.g. myFunction(page.Items)
}

答案 9 :(得分：0)

const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
    // optional tuning - 50% faster(cold) / 20% faster(hot)
    apiVersion: '2012-08-10',
    sslEnabled: false,
    paramValidation: false,
    convertResponseTypes: false,
    region: 'us-east-2' // put your region
});
const tableName = 'tableName'; // put your tablename

exports.handler = async (event, context, callback) => {
    let params = { TableName: tableName };

    let scanResults = [];
    let items;

    do {
        items = await docClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey = items.LastEvaluatedKey;
    } while (typeof items.LastEvaluatedKey != "undefined");

    callback(null, scanResults);
};

答案 10 :(得分：0)

这是扫描所有记录的直接替代品：

const scanAll = async (params) => {
    let all = [];
    while (true) {
        let data = await new Promise((resolve, reject) => {
            db.scan(params, function (err, data) {
                if (err)
                    reject(err);
                else
                    resolve(data);
            });
        });
        all = all.concat(data.Items);
        if (data.LastEvaluatedKey)
            params.ExclusiveStartKey = data.LastEvaluatedKey;
        else
            break;
    }
    return all;
};

用法：

scanAll(query)
    .catch((err) => {

    })
    .then((records) => {

    });
}

答案 11 :(得分：0)

scan方法读取表中的每个项目并返回表中的所有数据。您可以提供可选的filter_expression，以便仅返回与您的条件匹配的项目。但是，仅在扫描整个表格后才应用过滤器。 ref

我正在共享重构的onScan函数，希望对您有所帮助。

var AWS = require("aws-sdk");

AWS.config.update({
    region: "us-west-2",
    endpoint: "http://localhost:8000"
});

var docClient = new AWS.DynamoDB.DocumentClient();

async function read() {
        const params = {
            TableName: "tableName"
            // options can be passed here e.g.
            // FilterExpression: "#yr between :start_yr and :end_yr",
        };

        let items = [];
        return new Promise((resolve, reject) => {
            function onScan(err, data) {
                if (err) {
                    console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
                    reject();
                } else {
                    items = items.concat(data.Items);

                    // continue scanning if we have more items, because
                    // scan can retrieve a maximum of 1MB of data
                    if (typeof data.LastEvaluatedKey !== "undefined") {
                        params.ExclusiveStartKey = data.LastEvaluatedKey;
                        docClient.scan(params, onScan);
                    } else {
                        resolve(items);
                    }
                }
            }
            docClient.scan(params, onScan);
        });
    }
相关问题
最新问题