refactor(getDataDifference): 优化数据对比逻辑并提取公共方法

amos-boot-system-tzs/amos-boot-module-statistics/amos-boot-module-statistics-biz/src/main/java/com/yeejoin/amos/boot/module/statistcs/biz/service/impl/ComprehensiveStatisticalAnalysisServiceImpl.java

@@ -69,7 +69,6 @@ import org.springframework.core.io.Resource;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 import org.springframework.stereotype.Service;
 import org.springframework.util.ObjectUtils;
-import org.springframework.util.StopWatch;
 import org.springframework.web.multipart.MultipartFile;
 import org.typroject.tyboot.component.emq.EmqKeeper;
 import org.typroject.tyboot.core.foundation.context.RequestContext;
@@ -4131,8 +4130,9 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
     }
 
     public Object getDataDifference(String code) {
-        List<String> dataBase = new ArrayList<>();
-        List<String> esData = new ArrayList<>();
+        Map<String, Object> result = new HashMap<>();
+        List<String> dataBase;
+        List<String> esData;
         String index = StatisticalAnalysisEnum.getKey.get(code);
         List<String> dataBaseAllList = new ArrayList<>();
         List<String> esAllList = new ArrayList<>();
@@ -4141,63 +4141,149 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
             lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
             List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper);
             dataBaseAllList = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList());
-        } else if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
-            LambdaQueryWrapper<TzBaseEnterpriseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
-            lambdaQueryWrapper.select(TzBaseEnterpriseInfo::getUseCode);
-            lambdaQueryWrapper.eq(TzBaseEnterpriseInfo::getIsDelete, Boolean.FALSE);
-            List<TzBaseEnterpriseInfo> list = tzBaseEnterpriseInfoMapper.selectList(lambdaQueryWrapper);
-            dataBaseAllList = list.stream().map(TzBaseEnterpriseInfo::getUseCode).collect(Collectors.toList());
-        } else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
-            LambdaQueryWrapper<TzsUserInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
-            lambdaQueryWrapper.select(TzsUserInfo::getCertificateNum);
-            lambdaQueryWrapper.eq(TzsUserInfo::getIsDelete, Boolean.FALSE);
-            lambdaQueryWrapper.isNotNull(TzsUserInfo::getCertificateNum);
-            List<TzsUserInfo> tzsUserInfos = tzsUserInfoMapper.selectList(lambdaQueryWrapper);
-            dataBaseAllList = tzsUserInfos.stream().map(TzsUserInfo::getCertificateNum).collect(Collectors.toList());
+            esData = getEsData(Long.valueOf(list.size()), index, dataBaseAllList);
+            dataBase = getBaseData(Long.valueOf(list.size()), index, dataBaseAllList);
+        } else {
+            if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
+                LambdaQueryWrapper<TzBaseEnterpriseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
+                lambdaQueryWrapper.select(TzBaseEnterpriseInfo::getUseCode);
+                lambdaQueryWrapper.eq(TzBaseEnterpriseInfo::getIsDelete, Boolean.FALSE);
+                List<TzBaseEnterpriseInfo> list = tzBaseEnterpriseInfoMapper.selectList(lambdaQueryWrapper);
+                dataBaseAllList = list.stream().map(TzBaseEnterpriseInfo::getUseCode).collect(Collectors.toList());
+            } else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
+                LambdaQueryWrapper<TzsUserInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
+                lambdaQueryWrapper.select(TzsUserInfo::getCertificateNum);
+                lambdaQueryWrapper.eq(TzsUserInfo::getIsDelete, Boolean.FALSE);
+                lambdaQueryWrapper.isNotNull(TzsUserInfo::getCertificateNum);
+                List<TzsUserInfo> tzsUserInfos = tzsUserInfoMapper.selectList(lambdaQueryWrapper);
+                dataBaseAllList = tzsUserInfos.stream().map(TzsUserInfo::getCertificateNum).collect(Collectors.toList());
+            }
+            SearchRequest request = new SearchRequest();
+            request.indices(index);
+            SearchSourceBuilder builder = new SearchSourceBuilder();
+            BoolQueryBuilder boolMust = QueryBuilders.boolQuery();
+            boolMust.must(QueryBuilders.prefixQuery("superviseOrgCode", "50"));
+            builder.trackTotalHits(true);
+            builder.from(0);
+            builder.size(1000000);
+            request.source(builder);
+            try {
+                SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
+                for (SearchHit hit : response.getHits().getHits()) {
+                    JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
+                    JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
+                    if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
+                        esAllList.add(dto.getString("useCode"));
+                    } else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
+                        esAllList.add(dto.getString("certificateNum"));
+                    }
+                }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            esData = findElementsInANotInB(esAllList, dataBaseAllList);
+            dataBase = findElementsInANotInB(dataBaseAllList, esAllList);
         }
+        result.put("dataBase", dataBase);
+        result.put("esData", esData);
+        return result;
+    }
 
-        SearchRequest request = new SearchRequest();
-        request.indices(index);
-        SearchSourceBuilder builder = new SearchSourceBuilder();
-        builder.trackTotalHits(true);
-        builder.from(0);
-        builder.size(10000000);
-        request.source(builder);
-        try {
-            SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
-            for (SearchHit hit : response.getHits().getHits()) {
-                JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
-                JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
-                if (code.equals(StatisticalAnalysisEnum.equip.getCode())) {
-                    esAllList.add(dto.getString("id"));
-                } else if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
-                    esAllList.add(dto.getString("useCode"));
-                } else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
-                    esAllList.add(dto.getString("certificateNum"));
+    private List<String> getBaseData(Long count, String index, List<String> dataBaseAllList) {
+        List<String> result = new ArrayList<>();
+        Integer size = 10000;
+        Long times;
+        if (count != 0) {
+            times = count / size;
+            Long last = count % size;
+            if (last > 0) {
+                times++;
+            }
+        } else {
+            return result;
+        }
+        Long total = 0L;
+        for (int j = 0; j <= times; j++) {
+            // 按照size和j从dataBaseAllList中分批次取数据
+            if (j * size >= count) {
+                break;
+            }
+            total += (j + 1) * size - 1;
+            log.info("当前处理数据量：{}/{}", total, count);
+            List<String> dataBaseRecords = dataBaseAllList.subList(j * size, (j + 1) * size - 1 > count ? count.intValue() : (j + 1) * size - 1);
+            SearchRequest request = new SearchRequest();
+            request.indices(index);
+            SearchSourceBuilder builder = new SearchSourceBuilder();
+            builder.trackTotalHits(true);
+            BoolQueryBuilder boolMust = QueryBuilders.boolQuery();
+            boolMust.must(QueryBuilders.termsQuery("id", dataBaseRecords));
+            boolMust.must(QueryBuilders.prefixQuery("ORG_BRANCH_CODE", "50"));
+            boolMust.mustNot(QueryBuilders.termsQuery("STATUS", Arrays.asList("草稿", "已拒领", "待认领")));
+            builder.from(0);
+            builder.size(size);
+            request.source(builder);
+            List<String> esSearchRecords = new ArrayList<>();
+            try {
+                SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
+                for (SearchHit hit : response.getHits().getHits()) {
+                    JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
+                    JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
+                    esSearchRecords.add(dto.getString("id"));
                 }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
             }
-        } catch (IOException e) {
-            throw new RuntimeException(e);
+            result = findElementsInANotInB(dataBaseAllList, esSearchRecords);
         }
+        return result;
+    }
 
-        StopWatch watch1 = new StopWatch();
-        watch1.start();
-        esData = findElementsInANotInB(esAllList, dataBaseAllList);
-        watch1.stop();
-        log.warn("对比elementsInANotInB数据耗时：{}s", watch1.getTotalTimeSeconds());
+    private List<String> getEsData(Long count, String index, List<String> dataBaseAllList) {
+        Integer size = 10000;
+        List<String> result = new ArrayList<>();
+        Long times;
+        if (count != 0) {
+            times = count / size;
+            Long last = count % size;
+            if (last > 0) {
+                times++;
+            }
+        } else {
+            return result;
+        }
 
-        StopWatch watch2 = new StopWatch();
-        watch2.start();
-        dataBase = findElementsInANotInB(dataBaseAllList, esAllList);
-        watch2.stop();
-        log.warn("对比elementsInBNotInA数据耗时：{}s", watch2.getTotalTimeSeconds());
-        Map<String, Object> result = new HashMap<>();
-        result.put("dataBase", dataBase);
-        result.put("esData", esData);
+        for (int j = 0; j <= times; j++) {
+            SearchRequest request = new SearchRequest();
+            request.indices(index);
+            SearchSourceBuilder builder = new SearchSourceBuilder();
+            builder.trackTotalHits(true);
+            builder.from((j * size) >= count ? count.intValue() : (j * size));
+            builder.size(size);
+            request.source(builder);
+            List<String> esSearchRecords = new ArrayList<>();
+            try {
+                SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
+                for (SearchHit hit : response.getHits().getHits()) {
+                    JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
+                    JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
+                    esSearchRecords.add(dto.getString("SEQUENCE_NBR"));
+                }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            if (!ObjectUtils.isEmpty(esSearchRecords)) {
+                LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
+                lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
+                lambdaQueryWrapper.in(IdxBizJgUseInfo::getRecord, esSearchRecords);
+                List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper);
+                dataBaseAllList = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList());
+            }
+            List<String> elementsInANotInB = findElementsInANotInB(esSearchRecords, dataBaseAllList);
+            result.addAll(elementsInANotInB);
+        }
         return result;
     }
 
-
     // 找出AList中不在BList中的元素
     public static List<String> findElementsInANotInB(List<String> listA, List<String> listB) {
         Set<String> bSet = Collections.synchronizedSet(new HashSet<>(listB));