ProducerStream producing only to single partition

320 Views Asked by At

I am trying to produce some messages to a single topic having 2 partitions. All the messages are going to partition number 2 only. I would expect that a producer stream would distribute the messages across all partitions.

const kafka = require('kafka-node')
const { Transform } = require('stream');
const _ = require('lodash'); 
const client = new kafka.KafkaClient({ kafkaHost: 'localhost:9092' })
    , streamproducer = new kafka.ProducerStream({kafkaClient: client});

const stdinTransform = new Transform({
  objectMode: true,
  decodeStrings: true,
  transform (text, encoding, callback) {
    let num = parseInt(text);
    let message = { num: num, method: 'two' }
    console.log('pushing message')
    callback(null, {
      topic: 'topic356',
      messages: JSON.stringify(message)
    });
  }
});


stdinTransform.pipe(streamproducer);

function send() {
  var message = new Date().toString();
  stdinTransform.write([{ messages: [message] }]);
}
setInterval(send, 100);

ConsumerGroup:

var consumerOptions = {
  kafkaHost: '127.0.0.1:9092',
  groupId: 'ExampleTestGroup',
  sessionTimeout: 15000,
  protocol: ['roundrobin'],
  fromOffset: 'latest' // equivalent of auto.offset.reset valid values are 'none', 'latest', 'earliest'
};

var topics = 'topic356';

var consumerGroup = new ConsumerGroup(Object.assign({ id: 'consumer1' }, consumerOptions), topics);
consumerGroup.on('data', onMessage);

var consumerGroup2 = new ConsumerGroup(Object.assign({ id: 'consumer2' }, consumerOptions), topics);
consumerGroup2.on('data', onMessage);
consumerGroup2.on('connect', function () {
  setTimeout(function () {
    consumerGroup2.close(true, function (error) {
      console.log('consumer2 closed', error);
    });
  }, 25000);
});

function onMessage (message) {
  console.log(
    ` partition: ${message.partition} `
  );
}
2

There are 2 best solutions below

1
On

Do you produce messages with a key? In Kafka, messages with the same key are published to the same partition.

0
On

use partitionerType in options, the default is 0,

Partitioner type (default = 0, random = 1, cyclic = 2, keyed = 3, custom = 4), default 0

new kafka.Producer(new kafka.KafkaClient({ kafkaHost: 'localhost:9092' }),{
    partitionerType:1
});

https://github.com/SOHU-Co/kafka-node/issues/1094