Commit 91d1ea3e authored by tianbo's avatar tianbo

Merge branch 'develop_tzs_main' into develop_tzs_test

# Conflicts: # amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/controller/DataHandlerController.java # amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/service/impl/DataHandlerServiceImpl.java
parents 3f555ab7 b1050609
...@@ -449,4 +449,11 @@ public class DataHandlerController extends BaseController { ...@@ -449,4 +449,11 @@ public class DataHandlerController extends BaseController {
return ResponseHelper.buildResponse(dataHandlerService.updateMainNoticeJson()); return ResponseHelper.buildResponse(dataHandlerService.updateMainNoticeJson());
} }
@TycloudOperation(ApiLevel = UserType.AGENCY)
@PutMapping(value = "/insert/equip/db2newes")
@ApiOperation(httpMethod = "PUT", value = "1.添加数据库设备到ES新索引", notes = "添加数据库设备到ES新索引")
public ResponseModel<Integer> insertEquipFromDb2NewEs(@RequestBody Map<String, String> paramMap) {
return ResponseHelper.buildResponse(dataHandlerService.insertEquipFromDb2NewEs(paramMap));
}
} }
\ No newline at end of file
...@@ -528,7 +528,10 @@ public class ComprehensiveStatisticalAnalysisController extends BaseController { ...@@ -528,7 +528,10 @@ public class ComprehensiveStatisticalAnalysisController extends BaseController {
@TycloudOperation(ApiLevel = UserType.AGENCY, needAuth = false) @TycloudOperation(ApiLevel = UserType.AGENCY, needAuth = false)
@GetMapping(value = "/getDataDifferenceV2") @GetMapping(value = "/getDataDifferenceV2")
@ApiOperation(httpMethod = "GET", value = "获取es和数据库数据差异V2", notes = "获取es和数据库数据差异V2") @ApiOperation(httpMethod = "GET", value = "获取es和数据库数据差异V2", notes = "获取es和数据库数据差异V2")
public ResponseModel<Object> getDataDifferenceV2() { public ResponseModel<Object> getDataDifferenceV2(@RequestParam(value = "index") String index,
return ResponseHelper.buildResponse(statisticalAnalysisService.getDataDifferenceV2()); @RequestParam(value = "equListCode", required = false) String equListCode,
@RequestParam(value = "equCategoryCode", required = false) String equCategoryCode,
@RequestParam(value = "orgCode") String orgCode) {
return ResponseHelper.buildResponse(statisticalAnalysisService.getDataDifferenceV2(index, equListCode, equCategoryCode, orgCode));
} }
} }
...@@ -4715,44 +4715,60 @@ public class ComprehensiveStatisticalAnalysisServiceImpl { ...@@ -4715,44 +4715,60 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
/** /**
* 获取数据库与ES数据差异:找出ES比数据库多的记录ID * 获取数据库与ES数据差异:找出ES比数据库多的记录ID
*/ */
public Object getDataDifferenceV2() { public Object getDataDifferenceV2(String index, String equListCode, String equCategoryCode, String orgCode) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
List<String> extraInEs; List<String> extraInEs;
List<String> extraInDb;
// 获取索引
String index = "idx_biz_view_jg_all";
// 1. 查询数据库中所有设备 record // 1. 查询数据库中所有设备 record
//List<String> dbRecords = idxBizJgUseInfoMapper.selectEquipsClaimStatus(); //List<String> dbRecords = idxBizJgUseInfoMapper.selectEquipsClaimStatus();
LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>(); // LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord); // lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper); // lambdaQueryWrapper.eq(IdxBizJgUseInfo::getRecord, index);
List<String> dbRecords = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList()); List<String> dbRecords = idxBizJgUseInfoMapper.selectEquips(equListCode, equCategoryCode, orgCode);
log.info("数据库设备记录数: {}", dbRecords.size()); log.info("数据库设备记录数: {}", dbRecords.size());
// 2. 从ES中查询对应记录 // 2. 从ES中查询对应记录
List<String> esRecords = getEsDataV2(index); List<String> esRecords = getEsDataV2(index, equListCode, equCategoryCode, orgCode);
log.info("ES设备记录数: {}", esRecords.size()); log.info("ES设备记录数: {}", esRecords.size());
// 3. 对比差异(ES比DB多的记录) // 3. 对比差异(ES比DB多的记录)
extraInEs = findElementsInANotInB(esRecords, dbRecords); extraInEs = findElementsInANotInB(esRecords, dbRecords);
log.info("ES比数据库多的记录数: {}", extraInEs.size()); log.info("ES比数据库多的记录数: {}", extraInEs.size());
extraInDb = findElementsInANotInB(dbRecords, esRecords);
result.put("extraInEs", extraInEs); result.put("extraInEs", extraInEs);
result.put("extraIndb", extraInDb);
return result; return result;
} }
/** /**
* 从 ES 中拉取索引中所有的 SEQUENCE_NBR(match_all) * 从 ES 中拉取索引中所有的 SEQUENCE_NBR(match_all)
*/ */
private List<String> getEsDataV2(String index) { private List<String> getEsDataV2(String index, String equListCode, String equCategoryCode, String orgCode) {
// 使用 match_all 查询,拉取所有文档(通过 scroll 分批) // 使用 match_all 查询,拉取所有文档(通过 scroll 分批)
log.info("ES 全量查询开始, index={}, 时间={}", index, System.currentTimeMillis()); log.info("ES 全量查询开始, index={}, 时间={}", index, System.currentTimeMillis());
//BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
// .must(QueryBuilders.termsQuery("STATUS", Collections.singletonList("已认领")));
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.matchAllQuery()); if (index.equals("idx_biz_view_jg_all")){
boolQuery.must(QueryBuilders.wildcardQuery("ORG_BRANCH_CODE.keyword", QueryParser.escape(orgCode) + "*"));
}else{
boolQuery.must(QueryBuilders.wildcardQuery("ORG_BRANCH_CODE", QueryParser.escape(orgCode) + "*"));
}
// 设备种类编码
if (!ObjectUtils.isEmpty(equListCode)) {
BoolQueryBuilder elcBuilder = QueryBuilders.boolQuery();
elcBuilder.must(QueryBuilders.matchPhraseQuery("EQU_LIST_CODE", equListCode));
boolQuery.must(elcBuilder);
}
//设备类别编码
if (!ObjectUtils.isEmpty(equCategoryCode)) {
BoolQueryBuilder elcBuilder = QueryBuilders.boolQuery();
elcBuilder.must(QueryBuilders.matchPhraseQuery("EQU_CATEGORY_CODE", equCategoryCode));
boolQuery.must(elcBuilder);
}
// 线程安全的结果容器 // 线程安全的结果容器
List<String> esIds = Collections.synchronizedList(new ArrayList<>()); List<String> esIds = Collections.synchronizedList(new ArrayList<>());
try { try {
esSearchService.searchResponseInBatch(index, boolQuery, 2000, searchHits -> { esSearchService.searchResponseInBatch(index, boolQuery, 2000, searchHits -> {
// 每个批次只解析需要的字段 // 每个批次只解析需要的字段
......
...@@ -55,5 +55,7 @@ public interface IdxBizJgUseInfoMapper extends CustomBaseMapper<IdxBizJgUseInfo> ...@@ -55,5 +55,7 @@ public interface IdxBizJgUseInfoMapper extends CustomBaseMapper<IdxBizJgUseInfo>
List<String> selectEquipsClaimStatus(); List<String> selectEquipsClaimStatus();
List<String> selectEquips(@Param("equListCode") String equListCode, @Param("equCategoryCode") String equCategoryCode, @Param("orgCode") String orgCode);
List<String> selectUseInfoOfOneVersionWithParams(@Param("version") Integer version,@Param("params") Map<String, Object> params); List<String> selectUseInfoOfOneVersionWithParams(@Param("version") Integer version,@Param("params") Map<String, Object> params);
} }
...@@ -308,4 +308,17 @@ ...@@ -308,4 +308,17 @@
FROM idx_biz_jg_use_info ibjui FROM idx_biz_jg_use_info ibjui
JOIN idx_biz_jg_other_info joi ON ibjui."RECORD" = joi."RECORD" JOIN idx_biz_jg_other_info joi ON ibjui."RECORD" = joi."RECORD"
</select> </select>
<select id="selectEquips" resultType="java.lang.String">
SELECT s."RECORD" FROM "idx_biz_jg_supervision_info" s, idx_biz_jg_register_info r
where "ORG_BRANCH_CODE" like concat(#{orgCode},'%')
and s."RECORD" = r."RECORD"
<if test="equListCode != null and equListCode != ''">
AND r."EQU_LIST" = #{equListCode}
</if>
<if test="equCategoryCode != null and equCategoryCode != ''">
AND r."EQU_CATEGORY" = #{equCategoryCode}
</if>
</select>
</mapper> </mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment