kafka监控获取指定topic的消息总量示例


Posted in Python onDecember 23, 2019

我就废话不多说了,直接 上代码吧!

import kafka.api.PartitionOffsetRequestInfo;
import kafka.common.TopicAndPartition;
import kafka.javaapi.OffsetResponse;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataRequest;
import kafka.javaapi.consumer.SimpleConsumer;
 
import java.util.*;
import java.util.Map.Entry;
 
public class KafkaOffsetTools {
public final static String KAFKA_TOPIC_NAME_ADAPTER = "sample";
public final static String KAFKA_TOPIC_NAME_EXCEPTION = "exception";
public final static String KAFKA_TOPIC_NAME_AUDIT = "audit";
private static final String rawTopicTotal = "rawTopicTotalRecordCounter";
private static final String avroTopicTotal = "avroTopicTotalRecordCounter";
private static final String exceptionTopicTotal = "exceptionTopicTotalRecordCounter";
 
public KafkaOffsetTools() {
}
 
public static long getLastOffset(SimpleConsumer consumer, String topic,
int partition, long whichTime, String clientName) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
partition);
Map, PartitionOffsetRequestInfo> requestInfo = new HashMap, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
whichTime, 1));
kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
 
if (response.hasError()) {
System.err.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
return 0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
 
private TreeMap, PartitionMetadata> findLeader(List a_seedBrokers, String a_topic) {
TreeMap, PartitionMetadata> map = new TreeMap, PartitionMetadata>();
loop:
for (String seed : a_seedBrokers) {
SimpleConsumer consumer = null;
try {
String[] hostAndPort;
hostAndPort = seed.split(":");
consumer = new SimpleConsumer(hostAndPort[0], Integer.valueOf(hostAndPort[1]), 100000, 64 * 1024,
"leaderLookup" + new Date().getTime());
List topics = Collections.singletonList(a_topic);
TopicMetadataRequest req = new TopicMetadataRequest(topics);
kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
 
List metaData = resp.topicsMetadata();
for (TopicMetadata item : metaData) {
for (PartitionMetadata part : item.partitionsMetadata()) {
map.put(part.partitionId(), part);
}
}
} catch (Exception e) {
System.out.println("Error communicating with Broker [" + seed
+ "] to find Leader for [" + a_topic + ", ] Reason: " + e);
} finally {
if (consumer != null)
consumer.close();
}
}
return map;
}
 
public static void main(String[] args) {
String kafkaBrokerList = System.getenv("metadata.broker.list");
if(kafkaBrokerList == null || kafkaBrokerList.length() == 0){
System.err.println("No config kafka metadata.broker.list,it is null .");
//for test
kafkaBrokerList = "localhost:9092,localhost:9093";
System.err.println("Use this broker list for test,metadata.broker.list="+kafkaBrokerList);
}
//init topic,logSize = 0
Map,Integer> topics = new HashMap,Integer>();
topics.put(KAFKA_TOPIC_NAME_ADAPTER,0);
topics.put(KAFKA_TOPIC_NAME_EXCEPTION,0);
topics.put(KAFKA_TOPIC_NAME_AUDIT,0);
//init kafka broker list
String[] kafkaHosts;
kafkaHosts = kafkaBrokerList.split(",");
if (kafkaHosts == null || kafkaHosts.length == 0) {
System.err.println("No config kafka metadata.broker.list,it is null .");
System.exit(1);
}
List seeds = new ArrayList();
for (int i = 0; i < kafkaHosts.length; i++) {
seeds.add(kafkaHosts[i]);
}
 
KafkaOffsetTools kot = new KafkaOffsetTools();
 
for(String topicName : topics.keySet()){
TreeMap, PartitionMetadata> metadatas = kot.findLeader(seeds, topicName);
int logSize = 0;
for (Entry, PartitionMetadata> entry : metadatas.entrySet()) {
int partition = entry.getKey();
String leadBroker = entry.getValue().leader().host();
String clientName = "Client_" + topicName + "_" + partition;
SimpleConsumer consumer = new SimpleConsumer(leadBroker, entry.getValue().leader().port(), 100000,
64 * 1024, clientName);
long readOffset = getLastOffset(consumer, topicName, partition,
kafka.api.OffsetRequest.LatestTime(), clientName);
logSize += readOffset;
if (consumer != null) consumer.close();
}
topics.put(topicName,logSize);
}
System.out.println(topics.toString());
System.out.println(rawTopicTotal+"="+topics.get(KAFKA_TOPIC_NAME_ADAPTER)+" "+System.currentTimeMillis());
System.out.println(avroTopicTotal+"="+topics.get(KAFKA_TOPIC_NAME_AUDIT)+" "+System.currentTimeMillis());
System.out.println(exceptionTopicTotal+"="+topics.get(KAFKA_TOPIC_NAME_EXCEPTION)+" "+System.currentTimeMillis());
}
}

以上这篇kafka监控获取指定topic的消息总量示例就是小编分享给大家的全部内容了,希望能给大家一个参考,也希望大家多多支持三水点靠木。

Python 相关文章推荐
对pycharm代码整体左移和右移缩进快捷键的介绍
Jul 16 Python
Python基于多线程实现ping扫描功能示例
Jul 23 Python
Python正则表达式匹配日期与时间的方法
Jul 07 Python
详解Django定时任务模块设计与实践
Jul 24 Python
Python实现二叉搜索树BST的方法示例
Jul 30 Python
安装docker-compose的两种最简方法
Jul 30 Python
python sitk.show()与imageJ结合使用常见的问题
Apr 20 Python
基于python实现把json数据转换成Excel表格
May 07 Python
自学python用什么系统好
Jun 23 Python
python通用数据库操作工具 pydbclib的使用简介
Dec 21 Python
Python中Pyspider爬虫框架的基本使用详解
Jan 27 Python
Python中else的三种使用场景
Jun 16 Python
kafka-python 获取topic lag值方式
Dec 23 #Python
python3连接kafka模块pykafka生产者简单封装代码
Dec 23 #Python
python pyenv多版本管理工具的使用
Dec 23 #Python
Python测试Kafka集群(pykafka)实例
Dec 23 #Python
Python with语句和过程抽取思想
Dec 23 #Python
python每5分钟从kafka中提取数据的例子
Dec 23 #Python
Python彻底删除文件夹及其子文件方式
Dec 23 #Python
You might like
smarty section简介与用法分析
2008/10/03 PHP
laravel ORM 只开启created_at的几种方法总结
2018/01/29 PHP
犀利的js 函数集合
2009/06/11 Javascript
javascript学习笔记(十五) js间歇调用和超时调用
2012/06/20 Javascript
JQuery给元素添加/删除节点比如select
2013/04/02 Javascript
JS对话框_JS模态对话框showModalDialog用法总结
2014/01/11 Javascript
js验证电话号码与手机支持+86的正则表达式
2014/01/23 Javascript
浅谈JavaScript的内置对象和浏览器对象
2016/06/03 Javascript
利用select实现年月日三级联动的日期选择效果【推荐】
2016/12/13 Javascript
jsonp跨域请求实现示例
2017/03/13 Javascript
基于Vuejs和Element的注册插件的编写方法
2017/07/03 Javascript
Node.js爬取豆瓣数据实例分析
2018/03/05 Javascript
利用jquery和BootStrap实现动态滚动条效果
2018/12/03 jQuery
简单了解node npm cnpm的具体使用方法
2019/02/27 Javascript
JQuery获取元素尺寸、位置及页面滚动事件应用示例
2019/05/14 jQuery
laypage.js分页插件使用方法详解
2019/07/27 Javascript
[46:23]完美世界DOTA2联赛PWL S2 FTD vs Magma 第一场 11.20
2020/11/23 DOTA
Python Mysql数据库操作 Perl操作Mysql数据库
2009/01/12 Python
在Python的Django框架中更新数据库数据的方法
2015/07/17 Python
Python变量和字符串详解
2017/04/29 Python
Python3.5运算符操作实例详解
2019/04/25 Python
将Pytorch模型从CPU转换成GPU的实现方法
2019/08/19 Python
tensorflow-gpu安装的常见问题及解决方案
2020/01/20 Python
Pycharm安装并配置jupyter notebook的实现
2020/05/18 Python
Python 下载Bing壁纸的示例
2020/09/29 Python
python可视化 matplotlib画图使用colorbar工具自定义颜色
2020/12/07 Python
IE兼容css3圆角的实现代码
2011/07/21 HTML / CSS
工商管理本科毕业生求职信范文
2013/10/05 职场文书
医学实习生自我鉴定
2013/12/12 职场文书
KTV的创业计划书范文
2014/02/02 职场文书
公司委托书范本
2014/04/04 职场文书
应届生求职信范文
2014/05/26 职场文书
机关干部作风建设剖析材料
2014/10/23 职场文书
教师见习总结范文
2015/06/23 职场文书
Python实现机器学习算法的分类
2021/06/03 Python
开机音效回归! Windows 11重新引入开机铃声
2021/11/21 数码科技