Skip to content

A
amos-boot-biz
Commit 4a05d302 authored by helinlin's avatar helinlin
Browse files
Options

添加阿里云音频识别接口

parent f17dbfbf
Hide whitespace changes
Inline Side-by-side
Showing with 84 additions and 63 deletions
amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/AudioRecord.java
View file @ 4a05d302
......@@ -12,7 +12,7 @@ public class AudioRecord {
/**
* 说话角色
*/
private int type;
private String type;
/**
* 响应码 20000000 正常返回
*/
......
amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/AudioToText.java
View file @ 4a05d302
......@@ -9,7 +9,9 @@ import org.typroject.tyboot.component.emq.EmqKeeper;
import java.io.IOException;
import java.net.ServerSocket;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
/**
* 语音转文字
......@@ -32,16 +34,18 @@ public class AudioToText {
//启动两个监听端口监听推送进来的2路语音流
ServerSocket serverSocket1 = initServerSocketPort();
ServerSocket serverSocket2 = initServerSocketPort();
//识别的记录
List<AudioRecord> audioRecords = new ArrayList<>();
//识别的关键字
AudioKeyWord audioKeyWord = new AudioKeyWord();
logger.warn("myNumber监听的端口为：" + serverSocket1.getLocalPort() + " callerNumber监听的端口为：" + serverSocket2.getLocalPort());
//我的语音流
Thread thread1 = new Thread(() -> {
new AppSpeechTranscriber(new SpeechTranscriberListener(myNumber, myNumber, emqKeeper, audioKeyWord), serverSocket1).process();
new AppSpeechTranscriber(new SpeechTranscriberListener(myNumber, myNumber, emqKeeper, audioRecords, audioKeyWord), serverSocket1).process();
});
//呼入的语音流
Thread thread2 = new Thread(() -> {
new AppSpeechTranscriber(new SpeechTranscriberListener(callerNumber, myNumber, emqKeeper, audioKeyWord), serverSocket2).process();
new AppSpeechTranscriber(new SpeechTranscriberListener(myNumber, callerNumber, emqKeeper, audioRecords, audioKeyWord), serverSocket2).process();
});
thread1.setUncaughtExceptionHandler(new SubUncaughtExceptionHandler(serverSocket1));
thread2.setUncaughtExceptionHandler(new SubUncaughtExceptionHandler(serverSocket2));
......
amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/MessageType.java
View file @ 4a05d302
......@@ -26,10 +26,10 @@ public enum MessageType {
* 语音转文字关键字查找表
*/
enum MessageKeywords {
LOCATION("location", new String[]{"地址", "位置", "地点", "在"}),
LOCATION("location", new String[]{"地址", "位置", "地点",}),
CONTACT("contact", new String[]{"联系人", "联系", "在",}),
CONTACT_NUMBER("contactNumber", new String[]{"号码是", "号码", "手机是", "手机",}),
UNIT("unit", new String[]{"事发单位", "单位",}),
CONTACT_NUMBER("contactNumber", new String[]{"号码是", "号码", "手机是", "手机", "手机号", "手机号是"}),
UNIT("unit", new String[]{"事发单位", "单位", "部门"}),
PART("part", new String[]{"重点部位", "部位",});
private String type;
......
amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/SocketClient.java
View file @ 4a05d302
......@@ -19,9 +19,10 @@ import java.util.concurrent.TimeUnit;
public class SocketClient {
private static final Logger logger = LoggerFactory.getLogger(SocketClient.class);
private static final String[] testFilePath = {
"D:\\ffmpeg-4.4-full_build-shared\\bin\\out2.pcm",
"D:\\ffmpeg-4.4-full_build-shared\\bin\\out3.pcm",
"D:\\ffmpeg-4.4-full_build-shared\\bin\\out4.pcm",
"D:\\ffmpeg-4.4-full_build-shared\\bin\\record1.pcm",
"D:\\ffmpeg-4.4-full_build-shared\\bin\\record2.pcm",
};
public static void main(String[] args) {
......@@ -44,7 +45,7 @@ public class SocketClient {
while ((len = fis.read(b)) > 0) {
logger.info("send data pack length: " + len);
outputStream.write(b);
TimeUnit.MILLISECONDS.sleep(400);
TimeUnit.MILLISECONDS.sleep(200);
}
outputStream.flush();
outputStream.close();
......
amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/SpeechTranscriberListener.java
View file @ 4a05d302
......@@ -2,14 +2,14 @@ package com.yeejoin.amos.boot.module.jcs.biz.audioToText;
import com.alibaba.nls.client.protocol.asr.SpeechTranscriberResponse;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yeejoin.amos.component.rule.config.RuleConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.typroject.tyboot.component.emq.EmqKeeper;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 实时语音识别 回调函数
......@@ -17,19 +17,18 @@ import java.util.List;
public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.asr.SpeechTranscriberListener {
private static final Logger logger = LoggerFactory.getLogger(SpeechTranscriberListener.class);
public static final int RESULT_SUCCESS_CODE = 20000000;
//识别的记录
List<AudioRecord> audioRecords = new ArrayList<>();
private final String recordSendNumber;
private final String keywordSendNumber;
//mqtt客户端
private final String myNumber;
private final String typeNumber;
private final EmqKeeper emqKeeper;
private final AudioKeyWord audioKeyWord;
private final List<AudioRecord> audioRecords;
private final AudioKeyWord audioKeyWords;
public SpeechTranscriberListener(String recordSendNumber, String keywordSendNumber, EmqKeeper emqKeeper, AudioKeyWord audioKeyWord) {
this.recordSendNumber = recordSendNumber;
this.keywordSendNumber = keywordSendNumber;
public SpeechTranscriberListener(String myNumber, String typeNumber, EmqKeeper emqKeeper, List<AudioRecord> audioRecords, AudioKeyWord audioKeyWords) {
this.myNumber = myNumber;
this.typeNumber = typeNumber;
this.emqKeeper = emqKeeper;
this.audioKeyWord = audioKeyWord;
this.audioRecords = audioRecords;
this.audioKeyWords = audioKeyWords;
}
/**
......@@ -48,14 +47,10 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
", result: " + response.getTransSentenceText() +
//当前已处理的音频时长，单位为毫秒。
", time: " + response.getTransSentenceTime());
if (response.getStatus() == RESULT_SUCCESS_CODE) {
sendToMqtt(response, audioRecords);
extractKeyWord(response, audioKeyWord);
} else {
logger.error("异常的相应结果，响应码：" + response.getStatus());
}
responseHandler(response);
}
/**
* 服务端准备好了进行识别
*/
......@@ -99,6 +94,7 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
", begin_time: " + response.getSentenceBeginTime() +
//当前已处理的音频时长，单位为毫秒。
", time: " + response.getTransSentenceTime());
responseHandler(response);
}
/**
......@@ -124,37 +120,44 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
+ ", status_text: " + response.getStatusText());
}
private void responseHandler(SpeechTranscriberResponse response) {
if (response.getStatus() == RESULT_SUCCESS_CODE) {
AudioRecord audioRecord = new AudioRecord.AudioRecordBuilder()
.type(typeNumber)
.taskId(response.getTaskId())
.name(response.getName())
.status(response.getStatus())
.index(response.getTransSentenceIndex())
.message(response.getTransSentenceText())
.confidence(response.getConfidence())
.time(response.getTransSentenceTime())
.build();
int index = audioRecord.getIndex() - 1;
if (index >= audioRecords.size()) {
audioRecords.add(audioRecord);
} else {
audioRecords.set(index, audioRecord);
}
sendRecord(audioRecord);
sendKeyword();
} else {
logger.error("异常的相应结果，响应码：" + response.getStatus());
}
}
/**
* 异步发送结果至mqtt，保持回调函数畅通
*
* @param response 语音句子识别返回结果
* @param audioRecords 历史识别记录
* @param audioRecord 语音句子识别返回结果
*/
@Async
public void sendToMqtt(SpeechTranscriberResponse response, List<AudioRecord> audioRecords) {
AudioRecord audioRecord = new AudioRecord.AudioRecordBuilder()
.type(response.getTransSentenceIndex() % 2) // TODO 区别说话角色
.taskId(response.getTaskId())
.name(response.getName())
.status(response.getStatus())
.index(response.getTransSentenceIndex())
.message(response.getTransSentenceText())
.confidence(response.getConfidence())
.time(response.getTransSentenceTime())
.build();
int index = audioRecord.getIndex() - 1;
if (index >= audioRecords.size()) {
audioRecords.add(audioRecord);
} else {
audioRecords.set(index, audioRecord);
}
public void sendRecord(AudioRecord audioRecord) {
try {
ObjectMapper objectMapper = new ObjectMapper();
emqKeeper.getMqttClient().publish(
MessageType.RECORD.getName() + "/" + recordSendNumber,
MessageType.RECORD.getName() + "/" + myNumber,
objectMapper.writeValueAsString(audioRecord).getBytes(),
RuleConfig.DEFAULT_QOS, true);
1, true);
} catch (Exception e) {
e.printStackTrace();
logger.error("发送音频识别结果消息异常，原因：" + e.getMessage());
......@@ -163,31 +166,43 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
/**
* 异步发送结果至mqtt，保持回调函数畅通
*
* @param response 语音句子识别返回结果
* @param audioKeyWord //关键字结果集
*/
@Async
public void extractKeyWord(SpeechTranscriberResponse response, AudioKeyWord audioKeyWord) {
public void sendKeyword() {
try {
String result = response.getTransSentenceText();
//寻找关键字
StringBuilder stringBuilder = new StringBuilder();
audioRecords.forEach(audioRecord -> {
stringBuilder.append(",").append(audioRecord.getMessage());
});
String result = stringBuilder.toString();
//寻找电话号码
Pattern pattern = Pattern.compile("\\d{5,}\n");
Matcher matcher = pattern.matcher(result);
while (matcher.find()) {
audioKeyWords.getValues().get(MessageKeywords.CONTACT_NUMBER.getType()).add(matcher.group());
}
//寻找其他类型关键字
for (MessageKeywords messageKeyword : MessageKeywords.values()) {
if (messageKeyword == MessageKeywords.CONTACT_NUMBER) continue;
for (String keyword : messageKeyword.getKeyword()) {
int index = result.indexOf(keyword);
//TODO 暂时截取到末尾
if (index != -1) {
String keywordValue = result.substring(index);
audioKeyWord.getValues().get(messageKeyword.getType()).add(keywordValue);
int keywordIndex = result.indexOf(keyword);
//问答式：截取关键字，从关键字的下一个[,]号开始截取到下下一个[,]号结束
if (keywordIndex != -1) {
int keywordIndexNextFlag = result.indexOf(",", keywordIndex);
if (keywordIndexNextFlag == -1) continue;
int keywordIndexNextNextFlag = result.indexOf(",", keywordIndexNextFlag);
if (keywordIndexNextNextFlag == -1) continue;
String keywordValue = result.substring(keywordIndexNextFlag, keywordIndexNextNextFlag);
audioKeyWords.getValues().get(messageKeyword.getType()).add(keywordValue);
}
}
}
ObjectMapper objectMapper = new ObjectMapper();
byte[] bytes = objectMapper.writeValueAsString(audioKeyWord.getValues()).getBytes();
byte[] bytes = objectMapper.writeValueAsString(audioKeyWords.getValues()).getBytes();
emqKeeper.getMqttClient().publish(
MessageType.KEYWORD.getName() + "/" + keywordSendNumber,
MessageType.KEYWORD.getName() + "/" + myNumber,
bytes,
RuleConfig.DEFAULT_QOS, true);
1, true);
} catch (Exception e) {
e.printStackTrace();
logger.error("发送音频关键字消息异常，原因：" + e.getMessage());
......@@ -195,3 +210,4 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
}
}
amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/controller/Audio2TextController.java
View file @ 4a05d302
......@@ -62,8 +62,8 @@ public class Audio2TextController {
HashMap<String, Integer> convert = audioToText.convert(myNumber, callerNumber);
try {
TimeUnit.SECONDS.sleep(1);
socketClient.process(convert.get(myNumber), 0);
socketClient.process(convert.get(callerNumber), 1);
socketClient.process(convert.get(myNumber), 2);
socketClient.process(convert.get(callerNumber), 3);
} catch (InterruptedException e) {
e.printStackTrace();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment