Commit 8d2ad62a authored by 刘林's avatar 刘林

fix:(大屏):大屏数据处理接口修改

parent 63a7ed76
...@@ -442,4 +442,11 @@ public class DataHandlerController extends BaseController { ...@@ -442,4 +442,11 @@ public class DataHandlerController extends BaseController {
return ResponseHelper.buildResponse(dataHandlerService.synEquipFromDb2Es(equListCode, equCategoryCode, orgBranchCode)); return ResponseHelper.buildResponse(dataHandlerService.synEquipFromDb2Es(equListCode, equCategoryCode, orgBranchCode));
} }
@TycloudOperation(ApiLevel = UserType.AGENCY)
@PutMapping(value = "/insert/equip/db2newes")
@ApiOperation(httpMethod = "PUT", value = "1.添加数据库设备到ES新索引", notes = "添加数据库设备到ES新索引")
public ResponseModel<Integer> insertEquipFromDb2NewEs(@RequestBody Map<String, String> paramMap) {
return ResponseHelper.buildResponse(dataHandlerService.insertEquipFromDb2NewEs(paramMap));
}
} }
\ No newline at end of file
...@@ -528,7 +528,10 @@ public class ComprehensiveStatisticalAnalysisController extends BaseController { ...@@ -528,7 +528,10 @@ public class ComprehensiveStatisticalAnalysisController extends BaseController {
@TycloudOperation(ApiLevel = UserType.AGENCY, needAuth = false) @TycloudOperation(ApiLevel = UserType.AGENCY, needAuth = false)
@GetMapping(value = "/getDataDifferenceV2") @GetMapping(value = "/getDataDifferenceV2")
@ApiOperation(httpMethod = "GET", value = "获取es和数据库数据差异V2", notes = "获取es和数据库数据差异V2") @ApiOperation(httpMethod = "GET", value = "获取es和数据库数据差异V2", notes = "获取es和数据库数据差异V2")
public ResponseModel<Object> getDataDifferenceV2() { public ResponseModel<Object> getDataDifferenceV2(@RequestParam(value = "index") String index,
return ResponseHelper.buildResponse(statisticalAnalysisService.getDataDifferenceV2()); @RequestParam(value = "equListCode", required = false) String equListCode,
@RequestParam(value = "equCategoryCode", required = false) String equCategoryCode,
@RequestParam(value = "orgCode") String orgCode) {
return ResponseHelper.buildResponse(statisticalAnalysisService.getDataDifferenceV2(index, equListCode, equCategoryCode, orgCode));
} }
} }
...@@ -4715,44 +4715,60 @@ public class ComprehensiveStatisticalAnalysisServiceImpl { ...@@ -4715,44 +4715,60 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
/** /**
* 获取数据库与ES数据差异:找出ES比数据库多的记录ID * 获取数据库与ES数据差异:找出ES比数据库多的记录ID
*/ */
public Object getDataDifferenceV2() { public Object getDataDifferenceV2(String index, String equListCode, String equCategoryCode, String orgCode) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
List<String> extraInEs; List<String> extraInEs;
List<String> extraInDb;
// 获取索引
String index = "idx_biz_view_jg_all";
// 1. 查询数据库中所有设备 record // 1. 查询数据库中所有设备 record
//List<String> dbRecords = idxBizJgUseInfoMapper.selectEquipsClaimStatus(); //List<String> dbRecords = idxBizJgUseInfoMapper.selectEquipsClaimStatus();
LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>(); // LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord); // lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper); // lambdaQueryWrapper.eq(IdxBizJgUseInfo::getRecord, index);
List<String> dbRecords = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList()); List<String> dbRecords = idxBizJgUseInfoMapper.selectEquips(equListCode, equCategoryCode, orgCode);
log.info("数据库设备记录数: {}", dbRecords.size()); log.info("数据库设备记录数: {}", dbRecords.size());
// 2. 从ES中查询对应记录 // 2. 从ES中查询对应记录
List<String> esRecords = getEsDataV2(index); List<String> esRecords = getEsDataV2(index, equListCode, equCategoryCode, orgCode);
log.info("ES设备记录数: {}", esRecords.size()); log.info("ES设备记录数: {}", esRecords.size());
// 3. 对比差异(ES比DB多的记录) // 3. 对比差异(ES比DB多的记录)
extraInEs = findElementsInANotInB(esRecords, dbRecords); extraInEs = findElementsInANotInB(esRecords, dbRecords);
log.info("ES比数据库多的记录数: {}", extraInEs.size()); log.info("ES比数据库多的记录数: {}", extraInEs.size());
extraInDb = findElementsInANotInB(dbRecords, esRecords);
result.put("extraInEs", extraInEs); result.put("extraInEs", extraInEs);
result.put("extraIndb", extraInDb);
return result; return result;
} }
/** /**
* 从 ES 中拉取索引中所有的 SEQUENCE_NBR(match_all) * 从 ES 中拉取索引中所有的 SEQUENCE_NBR(match_all)
*/ */
private List<String> getEsDataV2(String index) { private List<String> getEsDataV2(String index, String equListCode, String equCategoryCode, String orgCode) {
// 使用 match_all 查询,拉取所有文档(通过 scroll 分批) // 使用 match_all 查询,拉取所有文档(通过 scroll 分批)
log.info("ES 全量查询开始, index={}, 时间={}", index, System.currentTimeMillis()); log.info("ES 全量查询开始, index={}, 时间={}", index, System.currentTimeMillis());
//BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
// .must(QueryBuilders.termsQuery("STATUS", Collections.singletonList("已认领")));
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.matchAllQuery()); if (index.equals("idx_biz_view_jg_all")){
boolQuery.must(QueryBuilders.wildcardQuery("ORG_BRANCH_CODE.keyword", QueryParser.escape(orgCode) + "*"));
}else{
boolQuery.must(QueryBuilders.wildcardQuery("ORG_BRANCH_CODE", QueryParser.escape(orgCode) + "*"));
}
// 设备种类编码
if (!ObjectUtils.isEmpty(equListCode)) {
BoolQueryBuilder elcBuilder = QueryBuilders.boolQuery();
elcBuilder.must(QueryBuilders.matchPhraseQuery("EQU_LIST_CODE", equListCode));
boolQuery.must(elcBuilder);
}
//设备类别编码
if (!ObjectUtils.isEmpty(equCategoryCode)) {
BoolQueryBuilder elcBuilder = QueryBuilders.boolQuery();
elcBuilder.must(QueryBuilders.matchPhraseQuery("EQU_CATEGORY_CODE", equCategoryCode));
boolQuery.must(elcBuilder);
}
// 线程安全的结果容器 // 线程安全的结果容器
List<String> esIds = Collections.synchronizedList(new ArrayList<>()); List<String> esIds = Collections.synchronizedList(new ArrayList<>());
try { try {
esSearchService.searchResponseInBatch(index, boolQuery, 2000, searchHits -> { esSearchService.searchResponseInBatch(index, boolQuery, 2000, searchHits -> {
// 每个批次只解析需要的字段 // 每个批次只解析需要的字段
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment