添加阿里云音频识别接口

amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/AudioRecord.java

@@ -12,7 +12,7 @@ public class AudioRecord {
     /**
      * 说话角色
      */
-    private int type;
+    private String type;
     /**
      * 响应码 20000000 正常返回
      */

amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/AudioToText.java

@@ -9,7 +9,9 @@ import org.typroject.tyboot.component.emq.EmqKeeper;
 
 import java.io.IOException;
 import java.net.ServerSocket;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 /**
  * 语音转文字
@@ -32,16 +34,18 @@ public class AudioToText {
         //启动两个监听端口监听推送进来的2路语音流
         ServerSocket serverSocket1 = initServerSocketPort();
         ServerSocket serverSocket2 = initServerSocketPort();
+        //识别的记录
+        List<AudioRecord> audioRecords = new ArrayList<>();
         //识别的关键字
         AudioKeyWord audioKeyWord = new AudioKeyWord();
         logger.warn("myNumber监听的端口为：" + serverSocket1.getLocalPort() + "  callerNumber监听的端口为：" + serverSocket2.getLocalPort());
         //我的语音流
         Thread thread1 = new Thread(() -> {
-            new AppSpeechTranscriber(new SpeechTranscriberListener(myNumber, myNumber, emqKeeper, audioKeyWord), serverSocket1).process();
+            new AppSpeechTranscriber(new SpeechTranscriberListener(myNumber, myNumber, emqKeeper, audioRecords, audioKeyWord), serverSocket1).process();
         });
         //呼入的语音流
         Thread thread2 = new Thread(() -> {
-            new AppSpeechTranscriber(new SpeechTranscriberListener(callerNumber, myNumber, emqKeeper, audioKeyWord), serverSocket2).process();
+            new AppSpeechTranscriber(new SpeechTranscriberListener(myNumber, callerNumber, emqKeeper, audioRecords, audioKeyWord), serverSocket2).process();
         });
         thread1.setUncaughtExceptionHandler(new SubUncaughtExceptionHandler(serverSocket1));
         thread2.setUncaughtExceptionHandler(new SubUncaughtExceptionHandler(serverSocket2));

amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/MessageType.java

@@ -26,10 +26,10 @@ public enum MessageType {
  * 语音转文字关键字查找表
  */
 enum MessageKeywords {
-    LOCATION("location", new String[]{"地址", "位置", "地点", "在"}),
+    LOCATION("location", new String[]{"地址", "位置", "地点",}),
     CONTACT("contact", new String[]{"联系人", "联系", "在",}),
-    CONTACT_NUMBER("contactNumber", new String[]{"号码是", "号码", "手机是", "手机",}),
-    UNIT("unit", new String[]{"事发单位", "单位",}),
+    CONTACT_NUMBER("contactNumber", new String[]{"号码是", "号码", "手机是", "手机", "手机号", "手机号是"}),
+    UNIT("unit", new String[]{"事发单位", "单位", "部门"}),
     PART("part", new String[]{"重点部位", "部位",});
 
     private String type;

amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/SocketClient.java

@@ -19,9 +19,10 @@ import java.util.concurrent.TimeUnit;
 public class SocketClient {
     private static final Logger logger = LoggerFactory.getLogger(SocketClient.class);
     private static final String[] testFilePath = {
-            "D:\\ffmpeg-4.4-full_build-shared\\bin\\out2.pcm",
             "D:\\ffmpeg-4.4-full_build-shared\\bin\\out3.pcm",
             "D:\\ffmpeg-4.4-full_build-shared\\bin\\out4.pcm",
+            "D:\\ffmpeg-4.4-full_build-shared\\bin\\record1.pcm",
+            "D:\\ffmpeg-4.4-full_build-shared\\bin\\record2.pcm",
     };
 
     public static void main(String[] args) {
@@ -44,7 +45,7 @@ public class SocketClient {
             while ((len = fis.read(b)) > 0) {
                 logger.info("send data pack length: " + len);
                 outputStream.write(b);
-                TimeUnit.MILLISECONDS.sleep(400);
+                TimeUnit.MILLISECONDS.sleep(200);
             }
             outputStream.flush();
             outputStream.close();

amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/audioToText/SpeechTranscriberListener.java

@@ -2,14 +2,14 @@ package com.yeejoin.amos.boot.module.jcs.biz.audioToText;
 
 import com.alibaba.nls.client.protocol.asr.SpeechTranscriberResponse;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yeejoin.amos.component.rule.config.RuleConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.scheduling.annotation.Async;
 import org.typroject.tyboot.component.emq.EmqKeeper;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * 实时语音识别 回调函数
@@ -17,19 +17,18 @@ import java.util.List;
 public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.asr.SpeechTranscriberListener {
     private static final Logger logger = LoggerFactory.getLogger(SpeechTranscriberListener.class);
     public static final int RESULT_SUCCESS_CODE = 20000000;
-    //识别的记录
-    List<AudioRecord> audioRecords = new ArrayList<>();
-    private final String recordSendNumber;
-    private final String keywordSendNumber;
-    //mqtt客户端
+    private final String myNumber;
+    private final String typeNumber;
     private final EmqKeeper emqKeeper;
-    private final AudioKeyWord audioKeyWord;
+    private final List<AudioRecord> audioRecords;
+    private final AudioKeyWord audioKeyWords;
 
-    public SpeechTranscriberListener(String recordSendNumber, String keywordSendNumber, EmqKeeper emqKeeper, AudioKeyWord audioKeyWord) {
-        this.recordSendNumber = recordSendNumber;
-        this.keywordSendNumber = keywordSendNumber;
+    public SpeechTranscriberListener(String myNumber, String typeNumber, EmqKeeper emqKeeper, List<AudioRecord> audioRecords, AudioKeyWord audioKeyWords) {
+        this.myNumber = myNumber;
+        this.typeNumber = typeNumber;
         this.emqKeeper = emqKeeper;
-        this.audioKeyWord = audioKeyWord;
+        this.audioRecords = audioRecords;
+        this.audioKeyWords = audioKeyWords;
     }
 
     /**
@@ -48,14 +47,10 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
                 ", result: " + response.getTransSentenceText() +
                 //当前已处理的音频时长，单位为毫秒。
                 ", time: " + response.getTransSentenceTime());
-        if (response.getStatus() == RESULT_SUCCESS_CODE) {
-            sendToMqtt(response, audioRecords);
-            extractKeyWord(response, audioKeyWord);
-        } else {
-            logger.error("异常的相应结果，响应码：" + response.getStatus());
-        }
+        responseHandler(response);
     }
 
+
     /**
      * 服务端准备好了进行识别
      */
@@ -99,6 +94,7 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
                 ", begin_time: " + response.getSentenceBeginTime() +
                 //当前已处理的音频时长，单位为毫秒。
                 ", time: " + response.getTransSentenceTime());
+        responseHandler(response);
     }
 
     /**
@@ -124,37 +120,44 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
                 + ", status_text: " + response.getStatusText());
     }
 
+    private void responseHandler(SpeechTranscriberResponse response) {
+        if (response.getStatus() == RESULT_SUCCESS_CODE) {
+            AudioRecord audioRecord = new AudioRecord.AudioRecordBuilder()
+                    .type(typeNumber)
+                    .taskId(response.getTaskId())
+                    .name(response.getName())
+                    .status(response.getStatus())
+                    .index(response.getTransSentenceIndex())
+                    .message(response.getTransSentenceText())
+                    .confidence(response.getConfidence())
+                    .time(response.getTransSentenceTime())
+                    .build();
+            int index = audioRecord.getIndex() - 1;
+            if (index >= audioRecords.size()) {
+                audioRecords.add(audioRecord);
+            } else {
+                audioRecords.set(index, audioRecord);
+            }
+            sendRecord(audioRecord);
+            sendKeyword();
+        } else {
+            logger.error("异常的相应结果，响应码：" + response.getStatus());
+        }
+    }
 
     /**
      * 异步发送结果至mqtt，保持回调函数畅通
      *
-     * @param response     语音句子识别返回结果
-     * @param audioRecords 历史识别记录
+     * @param audioRecord 语音句子识别返回结果
      */
     @Async
-    public void sendToMqtt(SpeechTranscriberResponse response, List<AudioRecord> audioRecords) {
-        AudioRecord audioRecord = new AudioRecord.AudioRecordBuilder()
-                .type(response.getTransSentenceIndex() % 2) // TODO 区别说话角色
-                .taskId(response.getTaskId())
-                .name(response.getName())
-                .status(response.getStatus())
-                .index(response.getTransSentenceIndex())
-                .message(response.getTransSentenceText())
-                .confidence(response.getConfidence())
-                .time(response.getTransSentenceTime())
-                .build();
-        int index = audioRecord.getIndex() - 1;
-        if (index >= audioRecords.size()) {
-            audioRecords.add(audioRecord);
-        } else {
-            audioRecords.set(index, audioRecord);
-        }
+    public void sendRecord(AudioRecord audioRecord) {
         try {
             ObjectMapper objectMapper = new ObjectMapper();
             emqKeeper.getMqttClient().publish(
-                    MessageType.RECORD.getName() + "/" + recordSendNumber,
+                    MessageType.RECORD.getName() + "/" + myNumber,
                     objectMapper.writeValueAsString(audioRecord).getBytes(),
-                    RuleConfig.DEFAULT_QOS, true);
+                    1, true);
         } catch (Exception e) {
             e.printStackTrace();
             logger.error("发送音频识别结果消息异常，原因：" + e.getMessage());
@@ -163,31 +166,43 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
 
     /**
      * 异步发送结果至mqtt，保持回调函数畅通
-     *
-     * @param response     语音句子识别返回结果
-     * @param audioKeyWord //关键字结果集
      */
     @Async
-    public void extractKeyWord(SpeechTranscriberResponse response, AudioKeyWord audioKeyWord) {
+    public void sendKeyword() {
         try {
-            String result = response.getTransSentenceText();
-            //寻找关键字
+            StringBuilder stringBuilder = new StringBuilder();
+            audioRecords.forEach(audioRecord -> {
+                stringBuilder.append(",").append(audioRecord.getMessage());
+            });
+            String result = stringBuilder.toString();
+            //寻找电话号码
+            Pattern pattern = Pattern.compile("\\d{5,}\n");
+            Matcher matcher = pattern.matcher(result);
+            while (matcher.find()) {
+                audioKeyWords.getValues().get(MessageKeywords.CONTACT_NUMBER.getType()).add(matcher.group());
+            }
+            //寻找其他类型关键字
             for (MessageKeywords messageKeyword : MessageKeywords.values()) {
+                if (messageKeyword == MessageKeywords.CONTACT_NUMBER) continue;
                 for (String keyword : messageKeyword.getKeyword()) {
-                    int index = result.indexOf(keyword);
-                    //TODO 暂时截取到末尾
-                    if (index != -1) {
-                        String keywordValue = result.substring(index);
-                        audioKeyWord.getValues().get(messageKeyword.getType()).add(keywordValue);
+                    int keywordIndex = result.indexOf(keyword);
+                    //问答式：截取关键字，从关键字的下一个[,]号开始截取到下下一个[,]号结束
+                    if (keywordIndex != -1) {
+                        int keywordIndexNextFlag = result.indexOf(",", keywordIndex);
+                        if (keywordIndexNextFlag == -1) continue;
+                        int keywordIndexNextNextFlag = result.indexOf(",", keywordIndexNextFlag);
+                        if (keywordIndexNextNextFlag == -1) continue;
+                        String keywordValue = result.substring(keywordIndexNextFlag, keywordIndexNextNextFlag);
+                        audioKeyWords.getValues().get(messageKeyword.getType()).add(keywordValue);
                     }
                 }
             }
             ObjectMapper objectMapper = new ObjectMapper();
-            byte[] bytes = objectMapper.writeValueAsString(audioKeyWord.getValues()).getBytes();
+            byte[] bytes = objectMapper.writeValueAsString(audioKeyWords.getValues()).getBytes();
             emqKeeper.getMqttClient().publish(
-                    MessageType.KEYWORD.getName() + "/" + keywordSendNumber,
+                    MessageType.KEYWORD.getName() + "/" + myNumber,
                     bytes,
-                    RuleConfig.DEFAULT_QOS, true);
+                    1, true);
         } catch (Exception e) {
             e.printStackTrace();
             logger.error("发送音频关键字消息异常，原因：" + e.getMessage());
@@ -195,3 +210,4 @@ public class SpeechTranscriberListener extends com.alibaba.nls.client.protocol.a
     }
 }
 
+

amos-boot-module/amos-boot-module-biz/amos-boot-module-jcs-biz/src/main/java/com/yeejoin/amos/boot/module/jcs/biz/controller/Audio2TextController.java

@@ -62,8 +62,8 @@ public class Audio2TextController {
         HashMap<String, Integer> convert = audioToText.convert(myNumber, callerNumber);
         try {
             TimeUnit.SECONDS.sleep(1);
-            socketClient.process(convert.get(myNumber), 0);
-            socketClient.process(convert.get(callerNumber), 1);
+            socketClient.process(convert.get(myNumber), 2);
+            socketClient.process(convert.get(callerNumber), 3);
         } catch (InterruptedException e) {
             e.printStackTrace();
         }