Confluent Kafka MM2 Offset translation

Ops Cluster Replication
1

Hello,

I’m using Confluent CE 7.4 and trying to replicate data from another cluster. The other cluster is apache kafka 2.2, it is not confluent. I am trying to replicate offsets along with messages and consumer groups using Kafka Connect and MM2. I managed to replicate data and consumer groups, but offsets aren’t being translated.

Is this a constraint so that I should use Replicator?

2

I’m posting the config of my connectors as well:

{
  "name": "mm2-checkpoint-connector",
  "connector.class": "org.apache.kafka.connect.mirror.MirrorCheckpointConnector",
  "clusters": "my-source,my-target",
  "source.cluster.alias": "my-source",
  "target.cluster.alias": "my-target",
  "target.cluster.bootstrap.servers": "X",
  "source.cluster.bootstrap.servers": "X",
  "tasks.max": "4",
  "key.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
  "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
  "header.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
  "replication.factor": "3",
  "checkpoints.topic.replication.factor": "3",
  "emit.checkpoints.interval.seconds": "20",
  "source.cluster.security.protocol": "PLAINTEXT",
  "target.cluster.security.protocol": "PLAINTEXT",
  "refresh.groups.interval.seconds": "30",
  "refresh.topics.interval.seconds": "30",
  "sync.group.offsets.enabled": "true",
  "emit.checkpoints.enabled": "true",
  "replication.policy.class": "org.apache.kafka.connect.mirror.IdentityReplicationPolicy",
  "sync.group.offsets.interval.seconds": "30",
  "topics": ".*",
  "groups": ".*",
  "offset-syncs.topic.location": "target",
  "config.storage.topic": "mirrormaker2-cluster-configs",
  "status.storage.topic": "mirrormaker2-cluster-status",
  "offset.storage.topic": "mirrormaker2-cluster-offsets"
}

{
  "name": "mm2-source-connector",
  "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector",
  "clusters": "my-source,my-target",
  "source.cluster.alias": "my-source",
  "target.cluster.alias": "my-target",
  "target.cluster.bootstrap.servers": "X",
  "source.cluster.bootstrap.servers": "X",
  "topics": ".*",
  "tasks.max": "4",
  "key.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
  "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
  "header.converter": "org.apache.kafka.connect.converters.ByteArrayConverter",
  "replication.factor": "3",
  "offset-syncs.topic.replication.factor": "3",
  "sync.topic.acls.interval.seconds": "20",
  "sync.topic.configs.interval.seconds": "20",
  "refresh.topics.interval.seconds": "20",
  "refresh.groups.interval.seconds": "20",
  "consumer.group.id": "mm2-mirror-source-consumer-group",
  "source.cluster.security.protocol": "PLAINTEXT",
  "target.cluster.security.protocol": "PLAINTEXT",
  "replication.policy.class": "org.apache.kafka.connect.mirror.IdentityReplicationPolicy",
  "groups": ".*",
  "offset-syncs.topic.location": "target",
  "config.storage.topic": "mirrormaker2-cluster-configs",
  "status.storage.topic": "mirrormaker2-cluster-status",
  "offset.storage.topic": "mirrormaker2-cluster-offsets"
}

Still isn’t replicating offsets. Tried with Kafka 2.7 as source as well.