Skip to content

A
amos-boot-biz
Commit e6f419dc authored by tianyiming's avatar tianyiming
Browse files
Options

refactor(getDataDifference): 优化数据对比逻辑并提取公共方法

parent 038d3aa8
Hide whitespace changes
Inline Side-by-side
Showing with 136 additions and 50 deletions
amos-boot-system-tzs/amos-boot-module-statistics/amos-boot-module-statistics-biz/src/main/java/com/yeejoin/amos/boot/module/statistcs/biz/service/impl/ComprehensiveStatisticalAnalysisServiceImpl.java
View file @ e6f419dc
......@@ -69,7 +69,6 @@ import org.springframework.core.io.Resource;
import org.springframework.data.elasticsearch.annotations.FieldType;
import org.springframework.stereotype.Service;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.multipart.MultipartFile;
import org.typroject.tyboot.component.emq.EmqKeeper;
import org.typroject.tyboot.core.foundation.context.RequestContext;
......@@ -4131,8 +4130,9 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
}
public Object getDataDifference(String code) {
List<String> dataBase = new ArrayList<>();
List<String> esData = new ArrayList<>();
Map<String, Object> result = new HashMap<>();
List<String> dataBase;
List<String> esData;
String index = StatisticalAnalysisEnum.getKey.get(code);
List<String> dataBaseAllList = new ArrayList<>();
List<String> esAllList = new ArrayList<>();
......@@ -4141,63 +4141,149 @@ public class ComprehensiveStatisticalAnalysisServiceImpl {
lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper);
dataBaseAllList = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList());
} else if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
LambdaQueryWrapper<TzBaseEnterpriseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(TzBaseEnterpriseInfo::getUseCode);
lambdaQueryWrapper.eq(TzBaseEnterpriseInfo::getIsDelete, Boolean.FALSE);
List<TzBaseEnterpriseInfo> list = tzBaseEnterpriseInfoMapper.selectList(lambdaQueryWrapper);
dataBaseAllList = list.stream().map(TzBaseEnterpriseInfo::getUseCode).collect(Collectors.toList());
} else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
LambdaQueryWrapper<TzsUserInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(TzsUserInfo::getCertificateNum);
lambdaQueryWrapper.eq(TzsUserInfo::getIsDelete, Boolean.FALSE);
lambdaQueryWrapper.isNotNull(TzsUserInfo::getCertificateNum);
List<TzsUserInfo> tzsUserInfos = tzsUserInfoMapper.selectList(lambdaQueryWrapper);
dataBaseAllList = tzsUserInfos.stream().map(TzsUserInfo::getCertificateNum).collect(Collectors.toList());
esData = getEsData(Long.valueOf(list.size()), index, dataBaseAllList);
dataBase = getBaseData(Long.valueOf(list.size()), index, dataBaseAllList);
} else {
if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
LambdaQueryWrapper<TzBaseEnterpriseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(TzBaseEnterpriseInfo::getUseCode);
lambdaQueryWrapper.eq(TzBaseEnterpriseInfo::getIsDelete, Boolean.FALSE);
List<TzBaseEnterpriseInfo> list = tzBaseEnterpriseInfoMapper.selectList(lambdaQueryWrapper);
dataBaseAllList = list.stream().map(TzBaseEnterpriseInfo::getUseCode).collect(Collectors.toList());
} else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
LambdaQueryWrapper<TzsUserInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(TzsUserInfo::getCertificateNum);
lambdaQueryWrapper.eq(TzsUserInfo::getIsDelete, Boolean.FALSE);
lambdaQueryWrapper.isNotNull(TzsUserInfo::getCertificateNum);
List<TzsUserInfo> tzsUserInfos = tzsUserInfoMapper.selectList(lambdaQueryWrapper);
dataBaseAllList = tzsUserInfos.stream().map(TzsUserInfo::getCertificateNum).collect(Collectors.toList());
}
SearchRequest request = new SearchRequest();
request.indices(index);
SearchSourceBuilder builder = new SearchSourceBuilder();
BoolQueryBuilder boolMust = QueryBuilders.boolQuery();
boolMust.must(QueryBuilders.prefixQuery("superviseOrgCode", "50"));
builder.trackTotalHits(true);
builder.from(0);
builder.size(1000000);
request.source(builder);
try {
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
for (SearchHit hit : response.getHits().getHits()) {
JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
esAllList.add(dto.getString("useCode"));
} else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
esAllList.add(dto.getString("certificateNum"));
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
esData = findElementsInANotInB(esAllList, dataBaseAllList);
dataBase = findElementsInANotInB(dataBaseAllList, esAllList);
}
result.put("dataBase", dataBase);
result.put("esData", esData);
return result;
}
SearchRequest request = new SearchRequest();
request.indices(index);
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.trackTotalHits(true);
builder.from(0);
builder.size(10000000);
request.source(builder);
try {
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
for (SearchHit hit : response.getHits().getHits()) {
JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
if (code.equals(StatisticalAnalysisEnum.equip.getCode())) {
esAllList.add(dto.getString("id"));
} else if (code.equals(StatisticalAnalysisEnum.company.getCode())) {
esAllList.add(dto.getString("useCode"));
} else if (code.equals(StatisticalAnalysisEnum.person.getCode())) {
esAllList.add(dto.getString("certificateNum"));
private List<String> getBaseData(Long count, String index, List<String> dataBaseAllList) {
List<String> result = new ArrayList<>();
Integer size = 10000;
Long times;
if (count != 0) {
times = count / size;
Long last = count % size;
if (last > 0) {
times++;
}
} else {
return result;
}
Long total = 0L;
for (int j = 0; j <= times; j++) {
// 按照size和j从dataBaseAllList中分批次取数据
if (j * size >= count) {
break;
}
total += (j + 1) * size - 1;
log.info("当前处理数据量：{}/{}", total, count);
List<String> dataBaseRecords = dataBaseAllList.subList(j * size, (j + 1) * size - 1 > count ? count.intValue() : (j + 1) * size - 1);
SearchRequest request = new SearchRequest();
request.indices(index);
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.trackTotalHits(true);
BoolQueryBuilder boolMust = QueryBuilders.boolQuery();
boolMust.must(QueryBuilders.termsQuery("id", dataBaseRecords));
boolMust.must(QueryBuilders.prefixQuery("ORG_BRANCH_CODE", "50"));
boolMust.mustNot(QueryBuilders.termsQuery("STATUS", Arrays.asList("草稿", "已拒领", "待认领")));
builder.from(0);
builder.size(size);
request.source(builder);
List<String> esSearchRecords = new ArrayList<>();
try {
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
for (SearchHit hit : response.getHits().getHits()) {
JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
esSearchRecords.add(dto.getString("id"));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} catch (IOException e) {
throw new RuntimeException(e);
result = findElementsInANotInB(dataBaseAllList, esSearchRecords);
}
return result;
}
StopWatch watch1 = new StopWatch();
watch1.start();
esData = findElementsInANotInB(esAllList, dataBaseAllList);
watch1.stop();
log.warn("对比elementsInANotInB数据耗时：{}s", watch1.getTotalTimeSeconds());
private List<String> getEsData(Long count, String index, List<String> dataBaseAllList) {
Integer size = 10000;
List<String> result = new ArrayList<>();
Long times;
if (count != 0) {
times = count / size;
Long last = count % size;
if (last > 0) {
times++;
}
} else {
return result;
}
StopWatch watch2 = new StopWatch();
watch2.start();
dataBase = findElementsInANotInB(dataBaseAllList, esAllList);
watch2.stop();
log.warn("对比elementsInBNotInA数据耗时：{}s", watch2.getTotalTimeSeconds());
Map<String, Object> result = new HashMap<>();
result.put("dataBase", dataBase);
result.put("esData", esData);
for (int j = 0; j <= times; j++) {
SearchRequest request = new SearchRequest();
request.indices(index);
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.trackTotalHits(true);
builder.from((j * size) >= count ? count.intValue() : (j * size));
builder.size(size);
request.source(builder);
List<String> esSearchRecords = new ArrayList<>();
try {
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
for (SearchHit hit : response.getHits().getHits()) {
JSONObject jsonObject = (JSONObject) JSONObject.toJSON(hit);
JSONObject dto = jsonObject.getJSONObject("sourceAsMap");
esSearchRecords.add(dto.getString("SEQUENCE_NBR"));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
if (!ObjectUtils.isEmpty(esSearchRecords)) {
LambdaQueryWrapper<IdxBizJgUseInfo> lambdaQueryWrapper = new LambdaQueryWrapper<>();
lambdaQueryWrapper.select(IdxBizJgUseInfo::getRecord);
lambdaQueryWrapper.in(IdxBizJgUseInfo::getRecord, esSearchRecords);
List<IdxBizJgUseInfo> list = idxBizJgUseInfoMapper.selectList(lambdaQueryWrapper);
dataBaseAllList = list.stream().map(IdxBizJgUseInfo::getRecord).collect(Collectors.toList());
}
List<String> elementsInANotInB = findElementsInANotInB(esSearchRecords, dataBaseAllList);
result.addAll(elementsInANotInB);
}
return result;
}
// 找出AList中不在BList中的元素
public static List<String> findElementsInANotInB(List<String> listA, List<String> listB) {
Set<String> bSet = Collections.synchronizedSet(new HashSet<>(listB));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment