Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
amos-boot-biz
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
项目统一框架
amos-boot-biz
Commits
5538a8ef
Commit
5538a8ef
authored
Sep 14, 2023
by
刘林
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fix(message):修改message转发kafka消息到emq为批量消息
parent
502e67e9
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
141 additions
and
65 deletions
+141
-65
Constant.java
...rc/main/java/com/yeejoin/amos/message/kafka/Constant.java
+8
-0
KafkaConsumerService.java
.../com/yeejoin/amos/message/kafka/KafkaConsumerService.java
+130
-64
application-dev.properties
...ils-message/src/main/resources/application-dev.properties
+3
-1
No files found.
amos-boot-utils/amos-boot-utils-message/src/main/java/com/yeejoin/amos/message/kafka/Constant.java
View file @
5538a8ef
...
...
@@ -7,4 +7,12 @@ package com.yeejoin.amos.message.kafka;
public
interface
Constant
{
String
INSERT
=
"INSERT"
;
String
UPDATE
=
"UPDATE"
;
String
DATA
=
"data"
;
String
TOPIC
=
"topic"
;
String
TABLE
=
"table"
;
String
TYPE
=
"type"
;
String
DB_TYPE
=
"dbType"
;
String
BODY
=
"body"
;
String
DATA_TYPE
=
"datatype"
;
String
STATE
=
"state"
;
}
amos-boot-utils/amos-boot-utils-message/src/main/java/com/yeejoin/amos/message/kafka/KafkaConsumerService.java
View file @
5538a8ef
package
com
.
yeejoin
.
amos
.
message
.
kafka
;
import
lombok.extern.slf4j.Slf4j
;
import
net.sf.json.JSONArray
;
import
net.sf.json.JSONObject
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.kafka.clients.consumer.ConsumerRecord
;
import
org.eclipse.paho.client.mqttv3.MqttException
;
import
org.springframework.beans.factory.annotation.Autowired
;
...
...
@@ -14,7 +12,9 @@ import org.springframework.stereotype.Service;
import
org.typroject.tyboot.component.emq.EmqKeeper
;
import
java.nio.charset.StandardCharsets
;
import
java.util.Arrays
;
import
java.util.List
;
import
java.util.Optional
;
import
static
com
.
yeejoin
.
amos
.
message
.
kafka
.
Constant
.*;
/**
* kafka 消费服务
...
...
@@ -28,89 +28,155 @@ public class KafkaConsumerService {
private
static
final
String
MQTT_TOPIC
=
"romaSite/data/transmit"
;
private
static
final
String
PROVINCE_MQTT_TOPIC
=
"province/data/transport"
;
@Value
(
"${system.zxj}"
)
private
boolean
isZxj
;
@Autowired
protected
EmqKeeper
emqKeeper
;
@Value
(
"${system.zxj}"
)
private
boolean
isZxj
;
/**
* 消费单条消息,topics 可以监听多个topic,如:topics = {"topic1", "topic2"}
* 批量消费kafka消息
* Kafka消息转emq
*
* @param message 消息
* @param consumerRecords messages
* @param ack ack
*/
@KafkaListener
(
id
=
"consumerSingle"
,
groupId
=
"zhTestGroup"
,
topics
=
"#{'${kafka.topics}'.split(',')}"
,
concurrency
=
"2"
)
public
void
consumerSingle
(
String
message
,
Acknowledgment
ack
)
{
JSONObject
messageObj
=
JSONObject
.
fromObject
(
message
);
@KafkaListener
(
id
=
"consumerSingle"
,
groupId
=
"zhTestGroup"
,
topics
=
"#{'${kafka.topics}'.split(',')}"
)
public
void
listen1
(
List
<
ConsumerRecord
<
String
,
String
>>
consumerRecords
,
Acknowledgment
ack
)
{
try
{
String
topic
=
null
;
JSONObject
data
=
null
;
if
(
messageObj
.
has
(
"topic"
))
{
topic
=
messageObj
.
getString
(
"topic"
);
data
=
messageObj
.
getJSONObject
(
"data"
);
emqKeeper
.
getMqttClient
().
publish
(
topic
,
data
.
toString
().
getBytes
(
StandardCharsets
.
UTF_8
),
0
,
false
);
ack
.
acknowledge
();
log
.
info
(
"消息转发成功"
+
messageObj
.
toString
());
for
(
ConsumerRecord
<
String
,
String
>
consumerRecord
:
consumerRecords
)
{
Optional
<?>
kafkaMessage
=
Optional
.
ofNullable
(
consumerRecord
.
value
());
if
(
kafkaMessage
.
isPresent
())
{
JSONObject
messageObj
=
JSONObject
.
fromObject
(
kafkaMessage
.
get
());
if
(
messageObj
.
has
(
TOPIC
))
{
emqKeeper
.
getMqttClient
().
publish
(
messageObj
.
optString
(
TOPIC
),
messageObj
.
getJSONObject
(
DATA
).
toString
()
.
getBytes
(
StandardCharsets
.
UTF_8
),
0
,
false
);
}
log
.
info
(
"kafka消费zhTestGroup消息{}"
,
messageObj
);
}
}
}
catch
(
Exception
e
)
{
log
.
error
(
"消息转发失败"
+
e
.
getMessage
(),
e
);
log
.
error
(
"kafka失败,当前失败的批次: data:{}"
,
consumerRecords
);
}
finally
{
ack
.
acknowledge
();
}
}
@KafkaListener
(
id
=
"kafkaRoma"
,
groupId
=
"kafkaRoma"
,
topics
=
"#{'${queue.kafka.topics}'.split(',')}"
,
containerFactory
=
"kafkaRomaContainerFactory"
)
public
void
kafkaListener
(
ConsumerRecord
<?,
String
>
record
,
Acknowledgment
ack
)
{
Optional
<?>
messages
=
Optional
.
ofNullable
(
record
.
value
());
if
(
messages
.
isPresent
())
{
/**
* 批量消费kafka消息
* 监听数据表变化kafka数据转发emq
*
* @param consumerRecords messages
* @param ack ack
*/
@KafkaListener
(
id
=
"provinceMessage"
,
groupId
=
"province"
,
topics
=
"#{'${kafka.topics}'.split(',')}"
)
public
void
listen
(
List
<
ConsumerRecord
<
String
,
String
>>
consumerRecords
,
Acknowledgment
ack
)
{
try
{
JSONObject
messageObj
=
JSONObject
.
fromObject
(
record
.
value
());
JSONObject
data
=
messageObj
.
getJSONObject
(
"body"
);
if
(
data
.
isEmpty
())
{
data
=
messageObj
;
data
.
put
(
"datatype"
,
"state"
);
if
(
isZxj
)
{
for
(
ConsumerRecord
<
String
,
String
>
consumerRecord
:
consumerRecords
)
{
Optional
<?>
kafkaMessage
=
Optional
.
ofNullable
(
consumerRecord
.
value
());
if
(
kafkaMessage
.
isPresent
())
{
JSONObject
messageObj
=
JSONObject
.
fromObject
(
kafkaMessage
.
get
());
String
type
=
messageObj
.
optString
(
TYPE
);
String
table
=
messageObj
.
optString
(
TABLE
);
if
(
Arrays
.
asList
(
INSERT
,
UPDATE
).
contains
(
type
)
&&
DBTableTypeEnum
.
have
(
table
)
!=
null
)
{
JSONObject
data
=
(
JSONObject
)
messageObj
.
getJSONArray
(
DATA
).
get
(
0
);
data
.
put
(
DB_TYPE
,
type
);
data
.
put
(
TABLE
,
table
);
emqKeeper
.
getMqttClient
().
publish
(
PROVINCE_MQTT_TOPIC
,
data
.
toString
().
getBytes
(
StandardCharsets
.
UTF_8
),
0
,
false
);
log
.
info
(
"kafka消费province消息{}"
,
messageObj
);
}
}
}
emqKeeper
.
getMqttClient
().
publish
(
MQTT_TOPIC
,
data
.
toString
().
getBytes
(
StandardCharsets
.
UTF_8
),
0
,
false
);
}
catch
(
MqttException
e
)
{
log
.
error
(
"消息转发失败"
+
e
.
getMessage
(),
e
);
}
}
catch
(
Exception
e
)
{
log
.
error
(
"kafka失败,当前失败的批次: data:{}"
,
consumerRecords
);
}
finally
{
ack
.
acknowledge
();
}
}
/**
*
省级消息转发
*
转发苏州,绍兴换流站Kafka数据对emq
*
* @param
message 省级消息
* @param
record record
* @param ack ack
*/
@KafkaListener
(
id
=
"provinceMessage"
,
groupId
=
"province"
,
topics
=
"#{'${kafka.topics}'.split(',')}"
,
concurrency
=
"2"
)
public
void
consumerSingle1
(
String
message
,
Acknowledgment
ack
)
{
log
.
info
(
"省级消息转发打印"
+
message
);
if
(
isZxj
)
{
Optional
<?>
messages
=
Optional
.
ofNullable
(
message
);
if
(
messages
.
isPresent
())
{
@KafkaListener
(
id
=
"kafkaRoma"
,
groupId
=
"kafkaRoma"
,
topics
=
"#{'${queue.kafka.topics}'.split(',')}"
,
containerFactory
=
"kafkaRomaContainerFactory"
)
public
void
kafkaListener
(
ConsumerRecord
<?,
String
>
record
,
Acknowledgment
ack
)
{
try
{
JSONObject
jsonObject
=
JSONObject
.
fromObject
(
message
);
String
type
=
jsonObject
.
optString
(
"type"
);
String
table
=
jsonObject
.
optString
(
"table"
);
if
(
Arrays
.
asList
(
Constant
.
INSERT
,
Constant
.
UPDATE
).
contains
(
type
)
&&
DBTableTypeEnum
.
have
(
table
)
!=
null
)
{
if
(
Arrays
.
asList
(
"INSERT"
,
"UPDATE"
).
contains
(
type
))
{
JSONArray
array
=
jsonObject
.
getJSONArray
(
"data"
);
JSONObject
data
=
(
JSONObject
)
array
.
get
(
0
);
data
.
put
(
"dbType"
,
type
);
data
.
put
(
"table"
,
table
);
emqKeeper
.
getMqttClient
().
publish
(
PROVINCE_MQTT_TOPIC
,
data
.
toString
().
getBytes
(
StandardCharsets
.
UTF_8
),
0
,
false
);
log
.
info
(
"省级消息: {}"
,
data
);
Optional
<?>
messages
=
Optional
.
ofNullable
(
record
.
value
());
if
(
messages
.
isPresent
())
{
JSONObject
messageObj
=
JSONObject
.
fromObject
(
record
.
value
());
if
(
messageObj
.
getJSONObject
(
BODY
).
isEmpty
())
{
messageObj
.
put
(
DATA_TYPE
,
STATE
);
}
emqKeeper
.
getMqttClient
().
publish
(
MQTT_TOPIC
,
messageObj
.
toString
().
getBytes
(
StandardCharsets
.
UTF_8
),
0
,
false
);
}
}
catch
(
MqttException
e
)
{
log
.
error
(
"消息转发
失败"
+
e
.
getMessage
(),
e
);
}
log
.
error
(
"换流站转发Kafka消息
失败"
+
e
.
getMessage
(),
e
);
}
finally
{
ack
.
acknowledge
();
}
}
}
///**
// * 消费单条消息,topics 可以监听多个topic,如:topics = {"topic1", "topic2"}
// *
// * @param message 消息
// */
//@KafkaListener(id = "consumerSingle", groupId = "zhTestGroup", topics = "#{'${kafka.topics}'.split(',')}", concurrency = "2")
//public void consumerSingle(String message, Acknowledgment ack) {
// JSONObject messageObj = JSONObject.fromObject(message);
// try {
// String topic = null;
// JSONObject data = null;
// if (messageObj.has("topic")) {
// topic = messageObj.getString("topic");
// data = messageObj.getJSONObject("data");
// emqKeeper.getMqttClient().publish(topic, data.toString().getBytes(StandardCharsets.UTF_8), 0, false);
// ack.acknowledge();
// log.info("消息转发成功" + messageObj);
// }
// } catch (Exception e) {
// log.error("消息转发失败" + e.getMessage(), e);
// }
//}
//
///**
// * 省级消息转发
// *
// * @param message 省级消息
// * @param ack ack
// */
//@KafkaListener(id = "provinceMessage", groupId = "province", topics = "#{'${kafka.topics}'.split(',')}", concurrency = "2")
//public void consumerSingle1(String message, Acknowledgment ack) {
// log.info("省级消息转发打印" + message);
// if(isZxj) {
// Optional<?> messages = Optional.ofNullable(message);
// if (messages.isPresent()) {
// try {
// JSONObject jsonObject = JSONObject.fromObject(message);
// String type = jsonObject.optString("type");
// String table = jsonObject.optString("table");
// if (Arrays.asList(Constant.INSERT, Constant.UPDATE).contains(type) && DBTableTypeEnum.have(table) != null) {
// if (Arrays.asList("INSERT", "UPDATE").contains(type)) {
// JSONArray array = jsonObject.getJSONArray("data");
// JSONObject data = (JSONObject)array.get(0);
// data.put("dbType", type);
// data.put("table", table);
// emqKeeper.getMqttClient().publish(PROVINCE_MQTT_TOPIC, data.toString().getBytes(StandardCharsets.UTF_8), 0, false);
// log.info("省级消息: {}", data);
// }
// }
// } catch (MqttException e) {
// log.error("消息转发失败" + e.getMessage(), e);
// }
// ack.acknowledge();
// }
// }
//}
/* @KafkaListener(id = "consumerBatch", topicPartitions = {
...
...
@@ -147,15 +213,15 @@ public class KafkaConsumerService {
//
//
//
//
//kafka的监听器,topic为"zhTest",消费者组为"zhTestGroup"
//
@KafkaListener(topics = "test", groupId = "zhTestGroup")
//
public void listenZhugeGroup(ConsumerRecord<String, String> record, Acknowledgment ack) {
//
String value = record.value();
//
System.out.println(value);
//
System.out.println(record);
//
//手动提交offset
//
ack.acknowledge();
//
}
//kafka的监听器,topic为"zhTest",消费者组为"zhTestGroup"
//
@KafkaListener(topics = "test", groupId = "zhTestGroup")
//
public void listenZhugeGroup(ConsumerRecord<String, String> record, Acknowledgment ack) {
//
String value = record.value();
//
System.out.println(value);
//
System.out.println(record);
//
//手动提交offset
//
ack.acknowledge();
//
}
}
amos-boot-utils/amos-boot-utils-message/src/main/resources/application-dev.properties
View file @
5538a8ef
...
...
@@ -67,7 +67,9 @@ spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.S
# \u624B\u52A8\u8C03\u7528Acknowledgment.acknowledge()\u540E\u7ACB\u5373\u63D0\u4EA4\uFF0C\u4E00\u822C\u4F7F\u7528\u8FD9\u79CD
# MANUAL_IMMEDIATE
spring.kafka.listener.ack-mode
=
manual_immediate
spring.kafka.listener.type
=
batch
spring.kafka.consumer.max-poll-records
=
1000
spring.kafka.consumer.fetch-max-wait
=
1000
management.health.redis.enabled
=
false
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment