Commit 91d1ea3e authored by tianbo's avatar tianbo

Merge branch 'develop_tzs_main' into develop_tzs_test

# Conflicts: # amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/controller/DataHandlerController.java # amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/service/impl/DataHandlerServiceImpl.java
parents 3f555ab7 b1050609
......@@ -449,4 +449,11 @@ public class DataHandlerController extends BaseController {
return ResponseHelper.buildResponse(dataHandlerService.updateMainNoticeJson());
}
@TycloudOperation(ApiLevel = UserType.AGENCY)
@PutMapping(value = "/insert/equip/db2newes")
@ApiOperation(httpMethod = "PUT", value = "1.添加数据库设备到ES新索引", notes = "添加数据库设备到ES新索引")
public ResponseModel<Integer> insertEquipFromDb2NewEs(@RequestBody Map<String, String> paramMap) {
return ResponseHelper.buildResponse(dataHandlerService.insertEquipFromDb2NewEs(paramMap));
}
}
\ No newline at end of file
......@@ -528,7 +528,10 @@ public class ComprehensiveStatisticalAnalysisController extends BaseController {
@TycloudOperation(ApiLevel = UserType.AGENCY, needAuth = false)
@GetMapping(value = "/getDataDifferenceV2")
@ApiOperation(httpMethod = "GET", value = "获取es和数据库数据差异V2", notes = "获取es和数据库数据差异V2")
public ResponseModel<Object> getDataDifferenceV2() {
return ResponseHelper.buildResponse(statisticalAnalysisService.getDataDifferenceV2());
public ResponseModel<Object> getDataDifferenceV2(@RequestParam(value = "index") String index,
@RequestParam(value = "equListCode", required = false) String equListCode,
@RequestParam(value = "equCategoryCode", required = false) String equCategoryCode,
@RequestParam(value = "orgCode") String orgCode) {
return ResponseHelper.buildResponse(statisticalAnalysisService.getDataDifferenceV2(index, equListCode, equCategoryCode, orgCode));
}
}
......@@ -4715,44 +4715,60 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
/**
* 获取数据库与ES数据差异:找出ES比数据库多的记录ID
*/
public Object getDataDifferenceV2() {
public Object getDataDifferenceV2(String index, String equListCode, String equCategoryCode, String orgCode) {
Map<String, Object> result = new HashMap<>();
List<String> extraInEs;
List<String> extraInDb;
// 获取索引
String index = "idx_biz_view_jg_all";
// 1. 查询数据库中所有设备 record
//List<String> dbRecords = idxBizJgUseInfoMapper.selectEquipsClaimStatus();
LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper);
List<String> dbRecords = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList());
// LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
// lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
// lambdaQueryWrapper.eq(IdxBizJgUseInfo::getRecord, index);
List<String> dbRecords = idxBizJgUseInfoMapper.selectEquips(equListCode, equCategoryCode, orgCode);
log.info("数据库设备记录数: {}", dbRecords.size());
// 2. 从ES中查询对应记录
List<String> esRecords = getEsDataV2(index);
List<String> esRecords = getEsDataV2(index, equListCode, equCategoryCode, orgCode);
log.info("ES设备记录数: {}", esRecords.size());
// 3. 对比差异(ES比DB多的记录)
extraInEs = findElementsInANotInB(esRecords, dbRecords);
log.info("ES比数据库多的记录数: {}", extraInEs.size());
extraInDb = findElementsInANotInB(dbRecords, esRecords);
result.put("extraInEs", extraInEs);
result.put("extraIndb", extraInDb);
return result;
}
/**
* 从 ES 中拉取索引中所有的 SEQUENCE_NBR(match_all)
*/
private List<String> getEsDataV2(String index) {
private List<String> getEsDataV2(String index, String equListCode, String equCategoryCode, String orgCode) {
// 使用 match_all 查询,拉取所有文档(通过 scroll 分批)
log.info("ES 全量查询开始, index={}, 时间={}", index, System.currentTimeMillis());
//BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
// .must(QueryBuilders.termsQuery("STATUS", Collections.singletonList("已认领")));
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.matchAllQuery());
if (index.equals("idx_biz_view_jg_all")){
boolQuery.must(QueryBuilders.wildcardQuery("ORG_BRANCH_CODE.keyword", QueryParser.escape(orgCode) + "*"));
}else{
boolQuery.must(QueryBuilders.wildcardQuery("ORG_BRANCH_CODE", QueryParser.escape(orgCode) + "*"));
}
// 设备种类编码
if (!ObjectUtils.isEmpty(equListCode)) {
BoolQueryBuilder elcBuilder = QueryBuilders.boolQuery();
elcBuilder.must(QueryBuilders.matchPhraseQuery("EQU_LIST_CODE", equListCode));
boolQuery.must(elcBuilder);
}
//设备类别编码
if (!ObjectUtils.isEmpty(equCategoryCode)) {
BoolQueryBuilder elcBuilder = QueryBuilders.boolQuery();
elcBuilder.must(QueryBuilders.matchPhraseQuery("EQU_CATEGORY_CODE", equCategoryCode));
boolQuery.must(elcBuilder);
}
// 线程安全的结果容器
List<String> esIds = Collections.synchronizedList(new ArrayList<>());
try {
esSearchService.searchResponseInBatch(index, boolQuery, 2000, searchHits -> {
// 每个批次只解析需要的字段
......
......@@ -55,5 +55,7 @@ public interface IdxBizJgUseInfoMapper extends CustomBaseMapper<IdxBizJgUseInfo>
List<String> selectEquipsClaimStatus();
List<String> selectEquips(@Param("equListCode") String equListCode, @Param("equCategoryCode") String equCategoryCode, @Param("orgCode") String orgCode);
List<String> selectUseInfoOfOneVersionWithParams(@Param("version") Integer version,@Param("params") Map<String, Object> params);
}
......@@ -308,4 +308,17 @@
FROM idx_biz_jg_use_info ibjui
JOIN idx_biz_jg_other_info joi ON ibjui."RECORD" = joi."RECORD"
</select>
<select id="selectEquips" resultType="java.lang.String">
SELECT s."RECORD" FROM "idx_biz_jg_supervision_info" s, idx_biz_jg_register_info r
where "ORG_BRANCH_CODE" like concat(#{orgCode},'%')
and s."RECORD" = r."RECORD"
<if test="equListCode != null and equListCode != ''">
AND r."EQU_LIST" = #{equListCode}
</if>
<if test="equCategoryCode != null and equCategoryCode != ''">
AND r."EQU_CATEGORY" = #{equCategoryCode}
</if>
</select>
</mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment