出现错误TopicsNotExistError:
“主题不存在topic_name”后,kafka中的消费停止消耗?如何仅在创建主题后才使用它,或者即使遇到错误也可以使其监听?
docker-compose.yml
version: "3.7"
services:
redis:
image: redis
restart: always
container_name: redis
expose:
- 6379
# volumes:
# - ./redis:/data
command:
bash -c 'redis-server --daemonize yes && /bin/bash'
tty: true
stdin_open: true
networks:
- pocket-bits
zookeeper:
image: wurstmeister/zookeeper:3.4.6
container_name: zookeeper
expose:
- "2181"
networks:
- pocket-bits
# volumes:
# - ./zookeeper:/tmp/zookeeper
kafka:
image: wurstmeister/kafka:2.11-2.0.0
container_name: kafka
depends_on:
- zookeeper
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
networks:
- pocket-bits
# volumes:
# - ./kafka_docker:/backup
mongodb:
image: mongo:latest
restart: always
container_name: mongodb
ports:
- "27017:27017"
command: mongod
# tty: true
# stdin_open: true
networks:
- pocket-bits
# volumes:
# - ./mongodb:/data
pocketbits_user_management:
build: ./userManagement
restart: always
container_name: pocketbits_user_management
ports:
- "3000:3000"
links:
- mongodb
- redis
- kafka
depends_on:
- mongodb
- redis
- kafka
networks:
- pocket-bits
volumes:
- ./userManagement:/usr/src/app
- /usr/src/app/node_modules
pocketbits_notification:
build: ./notifications
container_name: pocketbits_notification
restart: on-failure
#command: bash -c 'sleep 30'
ports:
- "3001:3001"
links:
- kafka
depends_on:
- pocketbits_user_management
networks:
- pocket-bits
volumes:
- ./notifications:/usr/src/app
- /usr/src/app/node_modules
networks:
pocket-bits:
signupController.js (producer in getOtp() function)
/* eslint-disable dot-notation */
/* eslint-disable consistent-return */
const bcrypt = require('bcrypt');
const redis = require('redis');
const Joi = require('@hapi/joi');
const kafka = require('kafka-node');
const config = require('../../config');
const User = require('../models/user');
const client = redis.createClient(config.redis.port, config.redis.host);
exports.getOtp = async (req, res) => {
const { Producer } = kafka;
const kClient = new kafka.KafkaClient({ kafkaHost: `${config.kafkaHost.host}:${config.kafkaHost.port}` });
const producer = new Producer(kClient);
const schema = Joi.object({
phoneNumber: Joi.string().pattern(/^(\+\d{1,2}\s)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}$/).required(),
});
try {
await schema.validateAsync(req.query);
const { phoneNumber } = req.query;
const otp = Math.floor(Math.random() * (10000 - 1000) + 1000).toString();
const data = {
phoneNumber,
otp,
};
const payloads = [
{ topic: 'topic1', messages: JSON.stringify(data), partition: 0 },
];
producer.on('ready', () => {
producer.send(payloads, (err, topicData) => {
if (topicData) {
client.setex(`otp-${phoneNumber}`, 60, otp);
res.status(200).json({
message: `OTP sent to ${phoneNumber}`,
status: 200,
request: {
type: 'POST',
url: 'http://localhost:3000/user/verifyOtp',
},
});
} else {
res.status(500).json({
message: 'Server error',
status: 500,
error: err,
});
}
});
});
producer.on('error', (err) => {
res.status(500).json({
message: 'Server error',
status: 500,
error: err,
});
});
} catch (err) {
return res.status(400).json({
error: err['details'][0].message,
message: 'Invalid input',
status: 400,
});
}
};
notifications.js(消费者)
const msg91 = require('msg91-sms');
const kafka = require('kafka-node');
const config = require('../../config');
exports.sendOtp = () => {
var Consumer = kafka.Consumer;
var client = new kafka.KafkaClient({ kafkaHost: `${config.kafkaHost.host}:${config.kafkaHost.port}` });
var consumer = new Consumer(
client,
[
{ topic: 'topic1', partition: 0, fromOffset: 'latest' }
],
{
autoCommit: true,
}
);
consumer.on('message', (message) => {
console.log("=============================");
const { phoneNumber, otp } = JSON.parse(message.value);
// msg91.sendOne(config.msg91.authKey, phoneNumber, otp, config.msg91.senderId, config.msg91.route, config.msg91.dialcode, (response) => {
// if (!response) {
// console.log("Error while sending OTP :", response);
// }
// });
});
consumer.on('error', (error) => {
console.log(error);
});
}
错误
TopicsNotExistError: The topic(s) topic1 do not exist
at new TopicsNotExistError (/usr/src/app/node_modules/kafka-node/lib/errors/TopicsNotExistError.js:11:9)
at /usr/src/app/node_modules/kafka-node/lib/kafkaClient.js:948:14
at /usr/src/app/node_modules/kafka-node/lib/kafkaClient.js:919:7
at /usr/src/app/node_modules/async/dist/async.js:3888:9
at /usr/src/app/node_modules/async/dist/async.js:473:16
at replenish (/usr/src/app/node_modules/async/dist/async.js:1006:25)
at iterateeCallback (/usr/src/app/node_modules/async/dist/async.js:995:17)
at /usr/src/app/node_modules/async/dist/async.js:969:16
at /usr/src/app/node_modules/async/dist/async.js:3885:13
at KafkaClient.wrappedFn (/usr/src/app/node_modules/kafka-node/lib/kafkaClient.js:481:14) {
topics: [ 'topic1' ],
message: 'The topic(s) topic1 do not exist'
}