Kafka集群简单测试

接上篇Linux系统Kafka集群搭建与简单测试，做简单的测试

新建common工程

• 新建listen-kafka-common工程，主要放置生产端和消费端公用部分代码
• 工程目录截图如下图
• 
• pom.xml文件如下

• <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
      <groupId>listen-kafka</groupId>
      <artifactId>listen-kafka-parent</artifactId>
      <version>0.0.1-SNAPSHOT</version>
    </parent>
    <packaging>jar</packaging>
    <artifactId>listen-kafka-common</artifactId>
    <dependencies>
    	<dependency>
  		<groupId>com.alibaba</groupId>
  		<artifactId>fastjson</artifactId>
  		<version>1.1.41</version>
  	</dependency>
  	<dependency>
  		<groupId>org.apache.kafka</groupId>
  		<artifactId>kafka_2.10</artifactId>
  		<version>0.10.0.0</version>
  	</dependency>
    </dependencies>
  </project>

  编写代码

• 实体类User

• package com.listen.kafka.common.entity;
  
  import java.io.Serializable;
  
  import com.alibaba.fastjson.JSONObject;
  
  public class User implements Serializable {
  
  	private String name;
  	private int age;
  	private boolean sex;
  	public User() {
  	}
  	public User(String name, int age, boolean sex) {
  		this.name = name;
  		this.age = age;
  		this.sex = sex;
  	}
  	
  	@Override
  	public String toString(){
  		return JSONObject.toJSONString(this);
  	}
  	public String getName() {
  		return name;
  	}
  	public void setName(String name) {
  		this.name = name;
  	}
  	public int getAge() {
  		return age;
  	}
  	public void setAge(int age) {
  		this.age = age;
  	}
  	public boolean isSex() {
  		return sex;
  	}
  	public void setSex(boolean sex) {
  		this.sex = sex;
  	}
  	
  	
  }
  

• 序列化类UserSerializer

• package com.listen.kafka.common.coder;
  
  import java.io.UnsupportedEncodingException;
  import java.util.Map;
  
  import org.apache.kafka.common.errors.SerializationException;
  import org.apache.kafka.common.serialization.Serializer;
  
  import com.listen.kafka.common.entity.User;
  
  public class UserSerializer implements Serializer<User> {
  
  	private String encoding = "UTF8";
  
  	@Override
  	public void configure(Map<String, ?> configs, boolean isKey) {
  		String propertyName = isKey ? "key.serializer.encoding" : "value.serializer.encoding";
  		Object encodingValue = configs.get(propertyName);
  		if (encodingValue == null){
  			encodingValue = configs.get("serializer.encoding");
  		}
  		if (encodingValue != null && encodingValue instanceof String){
  			encoding = (String) encodingValue;
  		}
  	}
  
  	@Override
  	public byte[] serialize(String topic, User data) {
  		try {
  			if (data == null)
  				return null;
  			else
  				return data.toString().getBytes(encoding);
  		} catch (UnsupportedEncodingException e) {
  			throw new SerializationException("Error when serializing T to byte[] due to unsupported encoding " + encoding);
  		}
  	}
  
  	@Override
  	public void close() {
  		//nothing to do
  	}
  
  
  }
  

• 反序列化类UserDeserializer

• package com.listen.kafka.common.coder;
  
  import java.io.UnsupportedEncodingException;
  import java.util.Map;
  
  import org.apache.kafka.common.errors.SerializationException;
  import org.apache.kafka.common.serialization.Deserializer;
  
  import com.alibaba.fastjson.JSONObject;
  import com.listen.kafka.common.entity.User;
  
  public class UserDeserializer implements Deserializer<User> {
  	private String encoding = "UTF8";
  
  	@Override
  	public void configure(Map<String, ?> configs, boolean isKey) {
  		String propertyName = isKey ? "key.deserializer.encoding" : "value.deserializer.encoding";
          Object encodingValue = configs.get(propertyName);
          if (encodingValue == null){
          	encodingValue = configs.get("deserializer.encoding");
          }
          if (encodingValue != null && encodingValue instanceof String){
          	encoding = (String) encodingValue;
          }
  	}
  
  	@Override
  	public User deserialize(String topic, byte[] data) {
  		 try {
  	            if (data == null){
  	            	return null;
  	            }
  	            else{
  	            	String json = new String(data, encoding);
  	            	User user = JSONObject.parseObject(json, User.class);
  	            	return user;
  	            }
  	        } catch (UnsupportedEncodingException e) {
  	            throw new SerializationException("Error when deserializing byte[] to User due to unsupported encoding " + encoding);
  	        }
  	}
  
  	@Override
  	public void close() {
  		//nothing to do
  	}
  
  }
  

• 新建生产消息工程listen-kafka-producer

• 工程目录结构如下图
• 
• pom.xml文件如下

• <?xml version="1.0"?>
  <project
  	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
  	xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  	<modelVersion>4.0.0</modelVersion>
  	<parent>
  		<groupId>listen-kafka</groupId>
  		<artifactId>listen-kafka-parent</artifactId>
  		<version>0.0.1-SNAPSHOT</version>
  	</parent>
  	<groupId>listen-kafka-producer</groupId>
  	<artifactId>listen-kafka-producer</artifactId>
  	<packaging>war</packaging>
  	<dependencies>
  		<dependency>
  			<groupId>junit</groupId>
  			<artifactId>junit</artifactId>
  			<version>3.8.1</version>
  			<scope>test</scope>
  		</dependency>
  		<dependency>
  			<groupId>javax.servlet</groupId>
  			<artifactId>servlet-api</artifactId>
  			<version>2.5</version>
  			<scope>provided</scope>
  		</dependency>
  		<dependency>
  			<groupId>org.apache.kafka</groupId>
  			<artifactId>kafka-clients</artifactId>
  			<version>0.10.0.0</version>
  		</dependency>
  		<dependency>
  			<groupId>listen-kafka</groupId>
  			<artifactId>listen-kafka-common</artifactId>
  			<version>0.0.1-SNAPSHOT</version>
  		</dependency>
  	</dependencies>
  	<build>
  		<finalName>listen-kafka-producer</finalName>
  	</build>
  </project>
  

• 生产消息类ListenKafkaProducer

• package com.listen.kafka.producer;
  
  import java.util.Properties;
  
  import org.apache.kafka.clients.producer.Callback;
  import org.apache.kafka.clients.producer.KafkaProducer;
  import org.apache.kafka.clients.producer.ProducerRecord;
  import org.apache.kafka.clients.producer.RecordMetadata;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import com.listen.kafka.common.entity.User;
  
  public class ListenKafkaProducer{
  
  	private static Logger log = LoggerFactory.getLogger(ListenKafkaProducer.class);
  
  	public static void main(String[] args) throws InterruptedException{
  		KafkaProducer<String, User> producer = initProducer();
  		int i = 0;
  		User user = null;
  		while(true){
  			user = new User("t-meng" + i, i, i%2 == 0 ? true : false);
  			ProducerRecord<String, User> record = new ProducerRecord<String, User>("t-meng", String.valueOf(i), user);
  			producer.send(record, new Callback() {
  				
  				@Override
  				public void onCompletion(RecordMetadata metadata, Exception exception) {
  					if (exception != null)
  						exception.printStackTrace();
  					System.out.println("message send to partition " + metadata.partition() + ", offset: " + metadata.offset());
  				}
  			});
  			i++;
  			Thread.sleep(1000);
  		}
  	}
  
  	private static KafkaProducer<String, User> initProducer(){
  		Properties props = new Properties();
  		props.put("bootstrap.servers", "192.168.75.141:9092,192.168.75.141:9093,192.168.75.141:9094");
  		props.put("acks", "all");
  		props.put("retries", 0);
  		props.put("batch.size", 16384);
  		props.put("linger.ms", 1);
  		props.put("buffer.memory", 33554432);
  //		props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
  //		props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
  		props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
  		props.put("value.serializer", "com.listen.kafka.common.coder.UserSerializer");
  		props.put("auto.create.topics.enable", true);
  
  		return new KafkaProducer<String, User>(props);
  	}
  }
  

• 新建消费消息工程listen-kafka-consumer

• 工程目录结构如下图
• 
• pom.xml文件，如下

• <?xml version="1.0"?>
  <project
  	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
  	xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  	<modelVersion>4.0.0</modelVersion>
  	<parent>
  		<groupId>listen-kafka</groupId>
  		<artifactId>listen-kafka-parent</artifactId>
  		<version>0.0.1-SNAPSHOT</version>
  	</parent>
  	<groupId>listen-kafka-consumer</groupId>
  	<artifactId>listen-kafka-consumer</artifactId>
  	<packaging>war</packaging>
  	<dependencies>
  		<dependency>
  			<groupId>junit</groupId>
  			<artifactId>junit</artifactId>
  			<version>3.8.1</version>
  			<scope>test</scope>
  		</dependency>
  		<dependency>
  			<groupId>javax.servlet</groupId>
  			<artifactId>servlet-api</artifactId>
  			<version>2.5</version>
  			<scope>provided</scope>
  		</dependency>
  		<dependency>
  			<groupId>org.apache.kafka</groupId>
  			<artifactId>kafka-clients</artifactId>
  			<version>0.10.0.0</version>
  		</dependency>
  		<dependency>
  			<groupId>org.apache.kafka</groupId>
  			<artifactId>kafka_2.10</artifactId>
  			<version>0.10.0.0</version>
  		</dependency>
  		<dependency>
  			<groupId>listen-kafka</groupId>
  			<artifactId>listen-kafka-common</artifactId>
  			<version>0.0.1-SNAPSHOT</version>
  		</dependency>
  	</dependencies>
  	<build>
  		<finalName>listen-kafka-consumer</finalName>
  	</build>
  </project>
  

• 消费消息类ListenKafkaConsumer

• package com.listen.kafka.consumer;
  
  import java.util.Arrays;
  import java.util.Properties;
  
  import org.apache.kafka.clients.consumer.ConsumerRecord;
  import org.apache.kafka.clients.consumer.ConsumerRecords;
  import org.apache.kafka.clients.consumer.KafkaConsumer;
  
  import com.listen.kafka.common.entity.User;
  
  public class ListenKafkaConsumer {
  	
  	public static void main(String[] args){
  		 KafkaConsumer<String, User> consumer = initConsumer();
  		 consumer.subscribe(Arrays.asList("t-meng"));
  		 while(true){
  			 ConsumerRecords<String, User> records = consumer.poll(100);
  			 for(ConsumerRecord<String, User> record : records){
  				 System.out.println("fetched from partition " + record.partition() + ", offset: " + record.offset() + ", message: " + record.value().toString());
  			 }
  		 }
  	}
  	
  	private static KafkaConsumer<String, User> initConsumer(){
  		Properties props = new Properties();
  	     props.put("bootstrap.servers", "192.168.75.141:9092,192.168.75.141:9093,192.168.75.141:9094");
  	     props.put("group.id", "group1");
  	     /**
  	      * latest 从消费到的最新的offset位置开始消费
  	      * earliest 从最初的offset位置开始消费，如果是新的topic话
  	      * none 从当前produce生产的最新的offset开始消费，默认
  	      */
  	     props.put("auto.offset.reset", "earliest");
  	     props.put("enable.auto.commit", "true");
  	     props.put("auto.commit.interval.ms", "1000");
  	     props.put("session.timeout.ms", "30000");
  //	     props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
  //	     props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
  	     props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
  	     props.put("value.deserializer", "com.listen.kafka.common.coder.UserDeserializer");
  	     return new KafkaConsumer<String, User>(props);
  	}
  	
  }
  

• 编译启动

• 编译3个工程，分别右键Run As --> Java Application 两个文件：ListenKafkaConsumer和ListenKafkaProducer
• 查看控制台打印的日志信息，可以看到消息正常的生产和消费，简单模拟消息的生产和消费。
• 生产的日志
•  
• 消费的日志
• 
• 码云源码地址：
• https://git.oschina.net/git.listen/listen-kafka-parent
• https://git.oschina.net/git.listen/listen-kafka-common
• https://git.oschina.net/git.listen/listen-kafka-producer
• https://git.oschina.net/git.listen/listen-kafka-consumer 

Over！ 

来源：oschina

链接：https://my.oschina.net/u/39617/blog/686191