我一直在尝试使用AWS kinesis。我无法以json字符串格式获取数据,因此我可以直接将其写入AWS
的elasticsearch服务并存储以供以后使用。
代码:
'use strict'
var env = require('dotenv').config();
var elasticsearch = require('elasticsearch');
var awsEs = require('http-aws-es');
let https = require('https');
let accessKey = 'xxxxxxxx';
let uri = 'xxxxxxx';
let path = 'xxxx';
var request = require('request');
var AWS = require('aws-sdk');
AWS.config.update({
accessKeyId: 'xxxxxx',
secretAccessKey: 'xxxxxx',
region: 'us-east-1'
});
var kinesis = new AWS.Kinesis({apiVersion: '2013-12-02'});
var iterator;
var params1 = {
ShardId: 'shardId-000000000001', /* required */
ShardIteratorType: 'LATEST', /* required */
StreamName: 'demo', /* required */
};
kinesis.getShardIterator(params1, function(err, data1) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data1); // successful response
});
var params = {
ShardIterator: kinesis.ShardIterator, /* required */
Limit: 5
};
var es = new elasticsearch.Client({
hosts: 'https://search-kafkaes-tos2gmttttteyeyeyeye.us-east-2.es.amazonaws.com',
connectionClass: awsEs,
amazonES: {
region: 'us-east-1',
accessKey: 'xxxxxx',
secretKey: 'xxxxxx'
}
});
var getJsonSentiment = function (query, callback) {
get_sentiments(query, function (data) {
callback(data)
});
}
//exports.handler = (event, context, callback) => {
kinesis.getRecords(params, function (err, data) {
if (err) {
console.log("Receive message error", err);
} else if (data.Messages) {
var jsonData = JSON.parse(data.Messages[0].Body)
var finalOutput = null;
var deleteParams = {
QueueUrl: queueURL,
ReceiptHandle: data.Messages[0].ReceiptHandle
};
var req = {
uri: 'https://wxxxxxxt',
method: 'POST',
headers: {
'Ocp-Apim-Subscription-Key': accessKey,
},
body: JSON.stringify({ 'documents': [{ 'id': '1', 'text': jsonData["text"] }] })
}
request(req, callback);
function callback(error, response, body) {
console.log("here3");
var dataJ = JSON.parse(body);
console.log("---------"+dataJ["documents"][0]["score"]);
console.log("topic:"+jsonData["topic"])
es.index({
index: 'tweets',
type: 'tweets',
body: {
topic: jsonData["topic"],
text: jsonData["text"],
latitude: jsonData["latitude"],
longitude: jsonData["longitude"],
score: dataJ["documents"][0]["score"]
}
});
}
错误日志:
Receive message error { MissingRequiredParameter: Missing required key 'ShardIterator' in params
at ParamValidator.fail (/Users/archit/Desktop/node_modules/aws-sdk/lib/param_validator.js:50:37)
at ParamValidator.validateStructure (/Users/archit/Desktop/node_modules/aws-sdk/lib/param_validator.js:61:14)
at ParamValidator.validateMember (/Users/archit/Desktop/node_modules/aws-sdk/lib/param_validator.js:88:21)
at ParamValidator.validate (/Users/archit/Desktop/node_modules/aws-sdk/lib/param_validator.js:34:10)
at Request.VALIDATE_PARAMETERS (/Users/archit/Desktop/node_modules/aws-sdk/lib/event_listeners.js:125:42)
at Request.callListeners (/Users/archit/Desktop/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at callNextListener (/Users/archit/Desktop/node_modules/aws-sdk/lib/sequential_executor.js:95:12)
at /Users/archit/Desktop/node_modules/aws-sdk/lib/event_listeners.js:85:9
at finish (/Users/archit/Desktop/node_modules/aws-sdk/lib/config.js:320:7)
at /Users/archit/Desktop/node_modules/aws-sdk/lib/config.js:338:9
message: 'Missing required key \'ShardIterator\' in params',
code: 'MissingRequiredParameter',
time: 2017-12-08T04:22:26.517Z }
{ ShardIterator: 'XXXXXxxxxx' }
我一直在尝试使用AWS kinesis。我无法以json格式获取数据,因此我可以直接将其写入AWS的elasticsearch服务并存储以供以后使用。