如何使消费者仅在kafka中创建主题后才能消费?

时间:2019-12-21 09:21:51

标签: node.js docker apache-kafka

出现错误TopicsNotExistError:“主题不存在topic_name”后,kafka中的消费停止消耗?如何仅在创建主题后才使用它,或者即使遇到错误也可以使其监听?

docker-compose.yml

version: "3.7"
services:
  redis:  
    image: redis
    restart: always
    container_name: redis
    expose:
      - 6379
    # volumes: 
    #   - ./redis:/data
    command:  
      bash -c 'redis-server --daemonize yes && /bin/bash'
    tty: true  
    stdin_open: true  
    networks:
      - pocket-bits

  zookeeper:
    image: wurstmeister/zookeeper:3.4.6
    container_name: zookeeper
    expose:
      - "2181"
    networks:
      - pocket-bits
    # volumes: 
      # - ./zookeeper:/tmp/zookeeper

  kafka:
    image: wurstmeister/kafka:2.11-2.0.0
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka   
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 
    networks:
      - pocket-bits
    # volumes: 
      # - ./kafka_docker:/backup       

  mongodb:
    image: mongo:latest
    restart: always
    container_name: mongodb
    ports:
      - "27017:27017"
    command: mongod
    # tty: true
    # stdin_open: true
    networks:
      - pocket-bits
    # volumes: 
    #   - ./mongodb:/data

  pocketbits_user_management:
    build: ./userManagement
    restart: always
    container_name: pocketbits_user_management
    ports:
      - "3000:3000"
    links:
      - mongodb
      - redis
      - kafka
    depends_on:
      - mongodb
      - redis
      - kafka
    networks:
      - pocket-bits
    volumes: 
      - ./userManagement:/usr/src/app
      - /usr/src/app/node_modules

  pocketbits_notification:
    build: ./notifications
    container_name: pocketbits_notification
    restart: on-failure
    #command: bash -c 'sleep 30'
    ports:
      - "3001:3001"     
    links:
      - kafka
    depends_on:
      - pocketbits_user_management
    networks:
      - pocket-bits
    volumes: 
      - ./notifications:/usr/src/app
      - /usr/src/app/node_modules
networks:
  pocket-bits:

signupController.js (producer in getOtp() function)

/* eslint-disable dot-notation */
/* eslint-disable consistent-return */
const bcrypt = require('bcrypt');
const redis = require('redis');
const Joi = require('@hapi/joi');
const kafka = require('kafka-node');
const config = require('../../config');

const User = require('../models/user');

const client = redis.createClient(config.redis.port, config.redis.host);




 exports.getOtp = async (req, res) => {
    const { Producer } = kafka;
    const kClient = new kafka.KafkaClient({ kafkaHost: `${config.kafkaHost.host}:${config.kafkaHost.port}` });
    const producer = new Producer(kClient);

    const schema = Joi.object({
      phoneNumber: Joi.string().pattern(/^(\+\d{1,2}\s)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}$/).required(),
    });

    try {
      await schema.validateAsync(req.query);
      const { phoneNumber } = req.query;
      const otp = Math.floor(Math.random() * (10000 - 1000) + 1000).toString();

      const data = {
        phoneNumber,
        otp,
      };

      const payloads = [
        { topic: 'topic1', messages: JSON.stringify(data), partition: 0 },
      ];

      producer.on('ready', () => {
        producer.send(payloads, (err, topicData) => {
          if (topicData) {
            client.setex(`otp-${phoneNumber}`, 60, otp);
            res.status(200).json({
              message: `OTP sent to ${phoneNumber}`,
              status: 200,
              request: {
                type: 'POST',
                url: 'http://localhost:3000/user/verifyOtp',
              },
            });
          } else {
            res.status(500).json({
              message: 'Server error',
              status: 500,
              error: err,
            });
          }
        });
      });

      producer.on('error', (err) => {
        res.status(500).json({
          message: 'Server error',
          status: 500,
          error: err,
        });
      });
    } catch (err) {
      return res.status(400).json({
        error: err['details'][0].message,
        message: 'Invalid input',
        status: 400,
      });
    }
  };

notifications.js(消费者)

const msg91 = require('msg91-sms');
const kafka = require('kafka-node');
const config = require('../../config');

exports.sendOtp = () => {
    var Consumer = kafka.Consumer;
    var client = new kafka.KafkaClient({ kafkaHost: `${config.kafkaHost.host}:${config.kafkaHost.port}` });
    var consumer = new Consumer(
        client,
        [
            { topic: 'topic1', partition: 0, fromOffset: 'latest' }
        ],
        {
            autoCommit: true,
        }
    );

    consumer.on('message', (message) => {
        console.log("=============================");
        const { phoneNumber, otp } = JSON.parse(message.value);
        // msg91.sendOne(config.msg91.authKey, phoneNumber, otp, config.msg91.senderId, config.msg91.route, config.msg91.dialcode, (response) => {
        //     if (!response) {
        //         console.log("Error while sending OTP :", response);
        //     }
        // }); 
    });

    consumer.on('error', (error) => {
        console.log(error);
    });
}

错误

TopicsNotExistError: The topic(s) topic1 do not exist
    at new TopicsNotExistError (/usr/src/app/node_modules/kafka-node/lib/errors/TopicsNotExistError.js:11:9)
    at /usr/src/app/node_modules/kafka-node/lib/kafkaClient.js:948:14
    at /usr/src/app/node_modules/kafka-node/lib/kafkaClient.js:919:7
    at /usr/src/app/node_modules/async/dist/async.js:3888:9
    at /usr/src/app/node_modules/async/dist/async.js:473:16
    at replenish (/usr/src/app/node_modules/async/dist/async.js:1006:25)
    at iterateeCallback (/usr/src/app/node_modules/async/dist/async.js:995:17)
    at /usr/src/app/node_modules/async/dist/async.js:969:16
    at /usr/src/app/node_modules/async/dist/async.js:3885:13
    at KafkaClient.wrappedFn (/usr/src/app/node_modules/kafka-node/lib/kafkaClient.js:481:14) {
  topics: [ 'topic1' ],
  message: 'The topic(s) topic1 do not exist'
}

0 个答案:

没有答案