Skip to content

A
amos-boot-biz
Commit c9772102 authored by suhuiguang's avatar suhuiguang
Browse files
Options

feat(jg)： 数据核对

1.数据同步接口性能优化
parent f2c6eeaa
Hide whitespace changes
Inline Side-by-side
Showing with 203 additions and 67 deletions
amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/controller/DataHandlerController.java
View file @ c9772102
......@@ -436,7 +436,7 @@ public class DataHandlerController extends BaseController {
@TycloudOperation(ApiLevel = UserType.AGENCY)
@PutMapping(value = "/sync/db2es/equipments")
@ApiOperation(httpMethod = "PUT", value = "2.同步设备数据到es新旧索引", notes = "同步设备数据到es新旧索引")
public ResponseModel<Integer> synEquipFromDb2Es(@ApiParam(value = "设备种类code") @RequestParam String equListCode,
public ResponseModel<Integer> syncEquipFromDb2Es(@ApiParam(value = "设备种类code") @RequestParam String equListCode,
@ApiParam(value = "设备类别code") @RequestParam(required = false) String equCategoryCode,
@ApiParam(value = "属地code") @RequestParam String orgBranchCode) {
return ResponseHelper.buildResponse(dataHandlerService.synEquipFromDb2Es(equListCode, equCategoryCode, orgBranchCode));
......
amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/data/fix/patcher/FilterableBatchDataPatcher.java
View file @ c9772102
......@@ -24,28 +24,41 @@ public abstract class FilterableBatchDataPatcher implements HistoricalDataPatche
IdxBizJgUseInfoServiceImpl useInfoService = applicationContext.getBean(IdxBizJgUseInfoServiceImpl.class);
Integer maxVersion = useInfoService.getBaseMapper().selectMaxVersionWithParams(buildFilter(params));
Integer nextVersion = maxVersion + 1;
List<String> refreshRecords = useInfoService.getBaseMapper().selectUseInfoOfOneVersionWithParams(nextVersion,buildFilter(params));
List<String> refreshRecords = useInfoService.getBaseMapper().selectUseInfoOfOneVersionWithParams(nextVersion, buildFilter(params));
int patchSize = refreshRecords.size();
while (!refreshRecords.isEmpty()) {
refreshRecords.parallelStream().forEach(record -> {
try {
beforePatching(record);
patchSingleRecord(record);
afterPatching(record);
} catch (Exception e) {
// 异常数据跳过
log.error("数据修补失败,设备:{}", record, e);
}
});
useInfoService.getBaseMapper().updateVersionBatch(refreshRecords, nextVersion);
refreshRecords = useInfoService.getBaseMapper().selectUseInfoOfOneVersionWithParams(nextVersion,buildFilter(params));
patchSize = patchSize + refreshRecords.size();
try {
refreshRecords.parallelStream().forEach(record -> {
try {
beforePatching(record);
patchSingleRecord(record);
afterPatching(record);
} catch (Exception e) {
// 异常数据跳过
log.error("单个方式数据修补失败,设备:{}", record, e);
}
});
patchBatchRecord(refreshRecords);
} catch (Exception e) {
// 本批次异常数据跳过
log.error("数据修补失败,设备:{}", refreshRecords, e);
} finally {
StopWatch watch1 = new StopWatch();
watch1.start();
useInfoService.getBaseMapper().updateVersionBatch(refreshRecords, nextVersion);
watch1.stop();
log.info("版本号批量更新条数：「{}」, 耗时：「{}」", refreshRecords.size(), watch1.getTotalTimeSeconds());
refreshRecords = useInfoService.getBaseMapper().selectUseInfoOfOneVersionWithParams(nextVersion, buildFilter(params));
patchSize = patchSize + refreshRecords.size();
}
}
watch.stop();
log.info("数据修补完成，共处理{}条记录，耗时: {}秒", patchSize, watch.getTotalTimeSeconds());
log.warn("数据修补完成，共处理{}条记录，耗时: {}秒", patchSize, watch.getTotalTimeSeconds());
return patchSize;
}
protected abstract void patchBatchRecord(List<String> refreshRecords);
protected abstract Map<String, Object> buildFilter(Map<String, Object> params);
protected abstract void beforePatching(String record);
......
amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/data/fix/service/FilterableEquipInsert2EsPatcher.java
View file @ c9772102
......@@ -4,22 +4,34 @@ import cn.hutool.core.bean.BeanUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.yeejoin.amos.boot.module.common.api.dao.ESEquipmentCategory;
import com.yeejoin.amos.boot.module.common.api.dto.ESEquipmentCategoryDto;
import com.yeejoin.amos.boot.module.common.api.entity.TzsDataRefreshMessage;
import com.yeejoin.amos.boot.module.common.biz.refresh.DataRefreshEvent;
import com.yeejoin.amos.boot.module.common.api.entity.ESEquipmentInfo;
import com.yeejoin.amos.boot.module.common.api.entity.EsEntity;
import com.yeejoin.amos.boot.module.jg.biz.data.fix.patcher.FilterableBatchDataPatcher;
import com.yeejoin.amos.boot.module.jg.biz.edit.utils.JsonDiffUtil;
import com.yeejoin.amos.boot.module.jg.biz.refresh.StatisticsDataUpdateService;
import com.yeejoin.amos.boot.module.jg.biz.refresh.handler.EquipmentRefreshHandler;
import com.yeejoin.amos.boot.module.jg.biz.service.impl.EsBulkService;
import com.yeejoin.amos.boot.module.ymt.api.entity.IdxBizJgInspectionDetectionInfo;
import com.yeejoin.amos.boot.module.ymt.api.entity.IdxBizJgMaintenanceRecordInfo;
import com.yeejoin.amos.boot.module.ymt.api.entity.IdxBizJgSupervisionInfo;
import com.yeejoin.amos.boot.module.ymt.api.mapper.IdxBizJgInspectionDetectionInfoMapper;
import com.yeejoin.amos.boot.module.ymt.api.mapper.IdxBizJgMaintenanceRecordInfoMapper;
import com.yeejoin.amos.boot.module.ymt.api.mapper.IdxBizJgSupervisionInfoMapper;
import com.yeejoin.amos.boot.module.ymt.api.mapper.IdxBizJgUseInfoMapper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StopWatch;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.yeejoin.amos.boot.module.jg.biz.service.impl.DataHandlerServiceImpl.IDX_BIZ_EQUIPMENT_INFO;
import static com.yeejoin.amos.boot.module.jg.biz.service.impl.DataHandlerServiceImpl.IDX_BIZ_VIEW_JG_ALL;
@Component
@Slf4j
......@@ -34,61 +46,163 @@ public class FilterableEquipInsert2EsPatcher extends FilterableBatchDataPatcher
private final IdxBizJgSupervisionInfoMapper idxBizJgSupervisionInfoMapper;
protected FilterableEquipInsert2EsPatcher(ApplicationContext applicationContext, ESEquipmentCategory equipmentCategory, EquipmentRefreshHandler refreshHandler, IdxBizJgUseInfoMapper idxBizJgUseInfoMapper, IdxBizJgSupervisionInfoMapper idxBizJgSupervisionInfoMapper) {
private final IdxBizJgMaintenanceRecordInfoMapper maintenanceRecordInfoMapper;
private final IdxBizJgInspectionDetectionInfoMapper inspectionDetectionInfoMapper;
private final EsBulkService esBulkService;
protected FilterableEquipInsert2EsPatcher(ApplicationContext applicationContext, ESEquipmentCategory equipmentCategory, EquipmentRefreshHandler refreshHandler, IdxBizJgUseInfoMapper idxBizJgUseInfoMapper, IdxBizJgSupervisionInfoMapper idxBizJgSupervisionInfoMapper, IdxBizJgMaintenanceRecordInfoMapper maintenanceRecordInfoMapper, IdxBizJgInspectionDetectionInfoMapper inspectionDetectionInfoMapper, EsBulkService esBulkService) {
super(applicationContext);
this.equipmentCategory = equipmentCategory;
this.refreshHandler = refreshHandler;
this.idxBizJgUseInfoMapper = idxBizJgUseInfoMapper;
this.idxBizJgSupervisionInfoMapper = idxBizJgSupervisionInfoMapper;
this.maintenanceRecordInfoMapper = maintenanceRecordInfoMapper;
this.inspectionDetectionInfoMapper = inspectionDetectionInfoMapper;
this.esBulkService = esBulkService;
}
@Override
protected Map<String, Object> buildFilter(Map<String, Object> params) {
return params;
protected void patchBatchRecord(List<String> refreshRecords) {
log.info("批量处理设备信息到es开始");
StopWatch watch = new StopWatch();
watch.start("批量查询设备信息");
List<Map<String, Object>> details = idxBizJgUseInfoMapper.queryDetailBatch(refreshRecords);
watch.stop();
Map<String, Map<String, Object>> recordDetailMap = details.stream().collect(Collectors.toMap(e -> (String) e.get("SEQUENCE_NBR"), Function.identity(), (k1, k2) -> k2));
watch.start("组装es设备老索引更新及新增的对象数据");
// 组装es设备老索引更新及新增的对象数据
List<ESEquipmentCategoryDto> esEquipmentCategoryDtos = getEsEquipmentCategoryDtos(refreshRecords, recordDetailMap);
watch.stop();
watch.start("组装es设备新索引更新及新增的对象数据");
// 组装es设备新索引更新及新增的对象数据
List<ESEquipmentInfo> esEquipmentInfos = getEsEquipmentInfos(refreshRecords, recordDetailMap);
watch.stop();
watch.start("es设备新旧索引保存");
// 多线程保存
List<CompletableFuture<Void>> futures = new ArrayList<>();
if (!esEquipmentCategoryDtos.isEmpty()) {
futures.add(CompletableFuture.runAsync(() -> {
StopWatch watch4 = new StopWatch();
watch4.start();
esBulkService.bulkUpsert(IDX_BIZ_VIEW_JG_ALL, esEquipmentCategoryDtos.stream().map(e -> new EsEntity<>(e.getSEQUENCE_NBR(), e)).collect(Collectors.toList()));
watch4.stop();
log.warn("[设备老索引] 批量入库 {} 条，耗时 {}s",
esEquipmentCategoryDtos.size(), watch4.getTotalTimeSeconds());
}));
}
if (!esEquipmentInfos.isEmpty()) {
futures.add(CompletableFuture.runAsync(() -> {
StopWatch watch4 = new StopWatch();
watch4.start();
esBulkService.bulkUpsert(IDX_BIZ_EQUIPMENT_INFO, esEquipmentInfos.stream().map(e -> new EsEntity<>(e.getSEQUENCE_NBR(), e)).collect(Collectors.toList()));
watch4.stop();
log.warn("[设备新索引] 批量入库 {} 条，耗时 {}s",
esEquipmentInfos.size(), watch4.getTotalTimeSeconds());
}));
}
// 等待所有任务完成（阻塞当前线程）
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
watch.stop();
log.warn("新索引数据补充匹配补充，总耗时情况：{}", watch.prettyPrint());
}
@Override
protected void beforePatching(String record) {
private List<ESEquipmentInfo> getEsEquipmentInfos(List<String> refreshRecords, Map<String, Map<String, Object>> recordDetailMap) {
// 设备最新的维保信息-维度：设备
List<IdxBizJgMaintenanceRecordInfo> lastMaintenanceRecordInfos = maintenanceRecordInfoMapper.selectLastedMainInfoBatch(refreshRecords);
Map<String, List<IdxBizJgMaintenanceRecordInfo>> recordLastMaintMap = lastMaintenanceRecordInfos.stream().collect(Collectors.groupingBy(IdxBizJgMaintenanceRecordInfo::getRecord));
}
// 设备、各检验类型下最新的检验信息-维度：设备、检验类型
List<IdxBizJgInspectionDetectionInfo> lastedInspectInfosGroupByInspectType = inspectionDetectionInfoMapper.selectLastedGroupByInspectTypeBatch(refreshRecords);
Map<String, List<IdxBizJgInspectionDetectionInfo>> recordInspectInfosGroupByInspectTypeMap = lastedInspectInfosGroupByInspectType.stream().collect(Collectors.groupingBy(IdxBizJgInspectionDetectionInfo::getRecord));
@Override
protected void patchSingleRecord(String record) {
Optional<ESEquipmentCategoryDto> op = equipmentCategory.findById(record);
// 插入旧索引
if (!op.isPresent()) {
// 设备最新的检验信息-维度：设备
Map<String, Optional<IdxBizJgInspectionDetectionInfo>> recordLastInspectionMap = lastedInspectInfosGroupByInspectType.stream().filter(e -> e.getNextInspectDate() != null).collect(Collectors.groupingBy(IdxBizJgInspectionDetectionInfo::getRecord, Collectors.maxBy(Comparator.comparing(IdxBizJgInspectionDetectionInfo::getNextInspectDate))));
List<ESEquipmentInfo> esEquipmentInfos = refreshRecords.parallelStream().map(record -> {
ESEquipmentInfo esEquipmentInfo = null;
try {
Map<String, Object> detail = idxBizJgUseInfoMapper.queryDetail(record);
ESEquipmentCategoryDto esEquipmentInfo = new ESEquipmentCategoryDto();
esEquipmentInfo = new ESEquipmentInfo();
Map<String, Object> detail = recordDetailMap.get(record);
StatisticsDataUpdateService.formatUseDate(detail);
BeanUtil.copyProperties(detail, esEquipmentInfo, true);
equipmentCategory.save(esEquipmentInfo);
// 最新检验信息-维度record
IdxBizJgInspectionDetectionInfo inspectionDetectionInfo = Optional.ofNullable(recordLastInspectionMap.get(record)).flatMap(i -> i).orElse(new IdxBizJgInspectionDetectionInfo());
// 最新维保信息-维度record
IdxBizJgMaintenanceRecordInfo lastMaintenanceRecordInfo = Optional.ofNullable(recordLastMaintMap.get(record)).filter(l -> !l.isEmpty()).map(list -> list.get(0)).orElse(new IdxBizJgMaintenanceRecordInfo());
// 最新检验信息-维度record、检验类型，存最新的一条
List<IdxBizJgInspectionDetectionInfo> inspectionDetectionInfos = recordInspectInfosGroupByInspectTypeMap.getOrDefault(record, new ArrayList<>());
StatisticsDataUpdateService.formatInspectDate(esEquipmentInfo, inspectionDetectionInfo, record);
esEquipmentInfo.setIssueDate(refreshHandler.getIssueDate(esEquipmentInfo.getUSE_ORG_CODE()));
esEquipmentInfo.setMAINTAIN_UNIT(lastMaintenanceRecordInfo.getMeUnitCreditCode());
esEquipmentInfo.setMAINTAIN_UNIT_NAME(lastMaintenanceRecordInfo.getMeUnitName());
esEquipmentInfo.setInspections(BeanUtil.copyToList(inspectionDetectionInfos, ESEquipmentInfo.Inspection.class));
esEquipmentInfo.setMaintenances(lastMaintenanceRecordInfo.getSequenceNbr() != null ? Collections.singletonList(BeanUtil.copyProperties(lastMaintenanceRecordInfo, ESEquipmentInfo.Maintenance.class)) : new ArrayList<>());
esEquipmentInfo.setTechParams(refreshHandler.buildTechParamByEquList(record, esEquipmentInfo.getEQU_LIST_CODE()));
if ("8000".equals(esEquipmentInfo.getEQU_LIST_CODE())) {
List<ESEquipmentInfo.TechParam> techParams = esEquipmentInfo.getTechParams();
List<ESEquipmentInfo.TechParam> pipeLength = techParams.stream().filter(e -> e.getParamKey().equals("pipeLength") && e.getDoubleValue() != null).collect(Collectors.toList());
if (!ObjectUtils.isEmpty(pipeLength)) {
esEquipmentInfo.setPipeLength(pipeLength.get(0).getDoubleValue());
}
}
} catch (Exception e) {
log.error("老设备索引插入处理失败：{}", record, e);
// 异常数据跳过
log.error("准备新设备索引数据失败：{}", record, e);
}
} else {
return esEquipmentInfo;
}).collect(Collectors.toList());
esEquipmentInfos.remove(null);
esEquipmentInfos = esEquipmentInfos.stream().filter(e -> StringUtils.isNotEmpty(e.getSEQUENCE_NBR())).collect(Collectors.toList());
return esEquipmentInfos;
}
private List<ESEquipmentCategoryDto> getEsEquipmentCategoryDtos(List<String> refreshRecords, Map<String, Map<String, Object>> recordDetailMap) {
List<ESEquipmentCategoryDto> esEquipmentCategoryDtos = refreshRecords.parallelStream().map(record -> {
ESEquipmentCategoryDto esEquipmentInfo = null;
Optional<ESEquipmentCategoryDto> op = equipmentCategory.findById(record);
try {
IdxBizJgSupervisionInfo supervisionInfo = idxBizJgSupervisionInfoMapper.selectOne(new LambdaQueryWrapper<IdxBizJgSupervisionInfo>().eq(IdxBizJgSupervisionInfo::getRecord, record)
.select(IdxBizJgSupervisionInfo::getRecord, IdxBizJgSupervisionInfo::getOrgBranchCode, IdxBizJgSupervisionInfo::getOrgBranchName));
ESEquipmentCategoryDto esEquipmentInfo = op.get();
if (supervisionInfo != null && StringUtils.isNotEmpty(supervisionInfo.getOrgBranchCode()) && JsonDiffUtil.isNullOrEmpty(esEquipmentInfo.getOrgBranchCode())) {
esEquipmentInfo.setOrgBranchCode(supervisionInfo.getOrgBranchCode());
esEquipmentInfo.setORG_BRANCH_NAME(supervisionInfo.getOrgBranchName());
equipmentCategory.save(esEquipmentInfo);
if (op.isPresent()) { // 存在更新
IdxBizJgSupervisionInfo supervisionInfo = idxBizJgSupervisionInfoMapper.selectOne(new LambdaQueryWrapper<IdxBizJgSupervisionInfo>().eq(IdxBizJgSupervisionInfo::getRecord, record)
.select(IdxBizJgSupervisionInfo::getRecord, IdxBizJgSupervisionInfo::getOrgBranchCode, IdxBizJgSupervisionInfo::getOrgBranchName));
esEquipmentInfo = op.get();
if (supervisionInfo != null && StringUtils.isNotEmpty(supervisionInfo.getOrgBranchCode()) && JsonDiffUtil.isNullOrEmpty(esEquipmentInfo.getOrgBranchCode())) {
esEquipmentInfo.setOrgBranchCode(supervisionInfo.getOrgBranchCode());
esEquipmentInfo.setORG_BRANCH_NAME(supervisionInfo.getOrgBranchName());
}
} else { // 不存在创建
Map<String, Object> detail = recordDetailMap.get(record);
esEquipmentInfo = new ESEquipmentCategoryDto();
StatisticsDataUpdateService.formatUseDate(detail);
BeanUtil.copyProperties(detail, esEquipmentInfo, true);
}
} catch (Exception e) {
log.error("老设备索引更新属地处理失败：{}", record, e);
log.error("准备老设备索引数据失败：{}", record, e);
}
}
// 插入或者新索引
try {
refreshHandler.doRefresh(new TzsDataRefreshMessage().setDataId(record).setOperation(DataRefreshEvent.Operation.INSERT.name()));
} catch (Exception e) {
log.error("新设备索引插入处理失败：{}", record, e);
}
return esEquipmentInfo;
}).collect(Collectors.toList());
esEquipmentCategoryDtos.remove(null);
return esEquipmentCategoryDtos;
}
@Override
protected Map<String, Object> buildFilter(Map<String, Object> params) {
return params;
}
@Override
protected void beforePatching(String record) {
}
@Override
protected void patchSingleRecord(String record) {
}
@Override
protected void afterPatching(String record) {
}
......
amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/refresh/handler/EquipmentRefreshHandler.java
View file @ c9772102
......@@ -88,11 +88,11 @@ public class EquipmentRefreshHandler implements IDataRefreshHandler {
}
private List<ESEquipmentInfo.TechParam> buildTechParamByEquList(String record, String equListCode) {
public List<ESEquipmentInfo.TechParam> buildTechParamByEquList(String record, String equListCode) {
return StringUtils.isNotEmpty(equListCode) ? statisticsDataUpdateService.getTechParams(equListCode, record) : new ArrayList<>();
}
private LocalDate getIssueDate(String useRegistrationCode) {
public LocalDate getIssueDate(String useRegistrationCode) {
if (StringUtils.isEmpty(useRegistrationCode)) {
return null;
}
......
amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/service/impl/DataHandlerServiceImpl.java
View file @ c9772102
......@@ -27,10 +27,7 @@ import com.yeejoin.amos.boot.module.common.api.dao.EsBaseEnterpriseInfoDao;
import com.yeejoin.amos.boot.module.common.api.dao.EsEquipmentDao;
import com.yeejoin.amos.boot.module.common.api.dao.EsUserInfoDao;
import com.yeejoin.amos.boot.module.common.api.dto.ESEquipmentCategoryDto;
import com.yeejoin.amos.boot.module.common.api.entity.ESEquipmentInfo;
import com.yeejoin.amos.boot.module.common.api.entity.EsBaseEnterpriseInfo;
import com.yeejoin.amos.boot.module.common.api.entity.EsUserInfo;
import com.yeejoin.amos.boot.module.common.api.entity.TzsUserPermission;
import com.yeejoin.amos.boot.module.common.api.entity.*;
import com.yeejoin.amos.boot.module.common.api.enums.CylinderTypeEnum;
import com.yeejoin.amos.boot.module.common.biz.refresh.cm.RefreshCmService;
import com.yeejoin.amos.boot.module.common.biz.service.impl.EsSearchServiceImpl;
......@@ -207,6 +204,8 @@ public class DataHandlerServiceImpl {
private final IdxBizJgUseInfoMapper idxBizJgUseInfoMapper;
private final EsBulkService esBulkService;
/**
* 安装告知压力管道历史数据修复-详情中的设备列表修改为汇总表格式
*
......@@ -2783,6 +2782,7 @@ public class DataHandlerServiceImpl {
if (!esEquipmentCategoryDtos.isEmpty()) {
totalUpdate = esEquipmentCategoryDtos.size() + totalUpdate;
esEquipmentCategory.saveAll(esEquipmentCategoryDtos);
esBulkService.bulkUpsert(IDX_BIZ_VIEW_JG_ALL, esEquipmentCategoryDtos.stream().map(e-> new EsEntity<>(e.getSEQUENCE_NBR(), e)).collect(Collectors.toList()));
idxBizJgSupervisionInfoServiceImpl.getBaseMapper().updateOrgBranchCodeBatch(esEquipmentCategoryDtos);
}
}
......
amos-boot-system-tzs/amos-boot-module-jg/amos-boot-module-jg-biz/src/main/java/com/yeejoin/amos/boot/module/jg/biz/service/impl/EsBulkService.java
View file @ c9772102
package com.yeejoin.amos.boot.module.jg.biz.service.impl;
import com.alibaba.fastjson.JSON;
import com.yeejoin.amos.boot.module.common.api.entity.EsEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
@Component
......@@ -22,15 +21,18 @@ public class EsBulkService {
private final RestHighLevelClient restHighLevelClient;
private final ElasticsearchConverter converter; // 依赖注入
public <T> void bulkUpsert(String index, List<EsEntity<T>> list) {
BulkRequest request = new BulkRequest();
list.forEach(item -> {
Map<String, Object> docData = converter.mapObject(item.getData());
request.add(new UpdateRequest(index, item.getId())
.doc(JSON.toJSONString(item.getData()), XContentType.JSON)
.upsert(JSON.toJSONString(item.getData()), XContentType.JSON));
.doc(docData)
.upsert(docData));
});
try {
// request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
} catch (Exception e) {
log.error("批量写入数据失败:{}", e.getMessage(), e);
......
amos-boot-system-tzs/amos-boot-module-ymt/amos-boot-module-ymt-api/src/main/resources/mapper/IdxBizJgUseInfoMapper.xml
View file @ c9772102
......@@ -79,7 +79,7 @@
<if test="params.orgBranchCode != null and params.orgBranchCode != ''">
and s."ORG_BRANCH_CODE" like concat(#{params.orgBranchCode},'%')
</if>
limit 10000
limit 1000
</select>
<select id="selectMaxVersion" resultType="java.lang.Integer">
SELECT
......@@ -120,9 +120,16 @@
WHERE ibjui."RECORD" = #{record}
</select>
<update id="updateVersionBatch">
<foreach collection="records" separator=";" item="record" open="" close="">
UPDATE idx_biz_jg_use_info SET "VERSION"=#{version} WHERE record = #{record}
</foreach>
UPDATE
idx_biz_jg_use_info
SET
"VERSION"=#{version}
WHERE
record = ANY(ARRAY[
<foreach collection="records" separator="," item="record" open="" close="">
#{record}
</foreach>
])
</update>
<sql id="equip-detail-es">
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment