Skip to content

A
amos-boot-biz
Commit a7bb0ad7 authored by liguofu@yeejoin.com's avatar liguofu@yeejoin.com
Browse files
Options

优化：【空工大】解析Excel文件，大量数据插入时，分批插入

parent a58f9b8f
Hide whitespace changes
Inline Side-by-side
Showing with 82 additions and 60 deletions
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/config/ClientHandler.java
View file @ a7bb0ad7
...@@ -8,11 +8,8 @@ import com.yeejoin.amos.component.robot.AmosRequestContext; ...@@ -8,11 +8,8 @@ import com.yeejoin.amos.component.robot.AmosRequestContext;
import com.yeejoin.amos.kgd.message.Constant; import com.yeejoin.amos.kgd.message.Constant;
import com.yeejoin.amos.kgd.message.model.TableFieldModel; import com.yeejoin.amos.kgd.message.model.TableFieldModel;
import com.yeejoin.amos.kgd.message.model.TableModel; import com.yeejoin.amos.kgd.message.model.TableModel;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.http.*; import org.springframework.http.*;
...@@ -31,6 +28,8 @@ import java.sql.Statement; ...@@ -31,6 +28,8 @@ import java.sql.Statement;
import java.util.*; import java.util.*;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/** /**
* @Author: xl * @Author: xl
...@@ -45,19 +44,19 @@ public class ClientHandler<path> implements Runnable { ...@@ -45,19 +44,19 @@ public class ClientHandler<path> implements Runnable {
private final String hostAndPort; private final String hostAndPort;
public ClientHandler(Socket socket, String hostAndPort) { private final String url;
this.socket = socket;
this.hostAndPort = hostAndPort; //127.0.0.1:30009
}
@Value("${spring.datasource.url}") private final String username;
private static String url = "jdbc:mysql://172.16.3.101:3306/jd_bearing?allowMultiQueries=true&serverTimezone=GMT%2B8&characterEncoding=utf8";
@Value("${spring.datasource.username}") private final String password;
private static String username = "root";
@Value("${spring.datasource.password}") public ClientHandler(Socket socket, String hostAndPort, String url, String username, String password) {
private static String password = "Yeejoin@2020"; this.socket = socket;
this.hostAndPort = hostAndPort; //127.0.0.1:30009
this.url = url; // 数据库连接地址
this.username = username;// 数据库用户名
this.password = password;// 数据库密码
}
public static final String DATABASE_NAME = "jd_bearing"; public static final String DATABASE_NAME = "jd_bearing";
/*String*/ /*String*/
...@@ -205,17 +204,25 @@ public class ClientHandler<path> implements Runnable { ...@@ -205,17 +204,25 @@ public class ClientHandler<path> implements Runnable {
} }
// 2.2插入数据 // 2.2插入数据
if (dataList.size() > 0) { if (dataList.size() > 0) {
for (int i = 0; i < dataList.size(); i++) { if (dataList.size() <= 1000) {
List<List<String>> lists; String insertSQL = insertDataSQLBatch(tableName, tableColumns, dataList);
if (dataList.size() > 1000 && i % 1000 == 0 && i > 0) { if (StringUtil.isNotEmpty(insertSQL)) {
lists = (i + 1000) < dataList.size() ? dataList.subList(i, i + 1000) : dataList.subList(i, dataList.size()); state.execute(insertSQL);
}else {
lists = dataList;
} }
if(lists.size()>0){ } else {
String insertSQL = insertDataSQLBatch(tableName, tableColumns, lists); for (int i = 0; i <= dataList.size(); i++) {
if (StringUtil.isNotEmpty(insertSQL)) { List<List<String>> lists = null;
state.execute(insertSQL); if (i == dataList.size()) {
lists = dataList.subList(dataList.size() - i % 1000, dataList.size());
}
if (i % 1000 == 0 && i > 0) {
lists = dataList.subList(i - 1000, i);
}
if (lists != null) {
String insertSQL = insertDataSQLBatch(tableName, tableColumns, lists);
if (StringUtil.isNotEmpty(insertSQL)) {
state.execute(insertSQL);
}
} }
} }
} }
...@@ -249,7 +256,7 @@ public class ClientHandler<path> implements Runnable { ...@@ -249,7 +256,7 @@ public class ClientHandler<path> implements Runnable {
/** /**
* 连接MySQL数据库 * 连接MySQL数据库
*/ */
public static Map<String, Object> connectMySQL() { public Map<String, Object> connectMySQL() {
Connection connection = null; Connection connection = null;
Statement statement = null; Statement statement = null;
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
...@@ -383,56 +390,38 @@ public class ClientHandler<path> implements Runnable { ...@@ -383,56 +390,38 @@ public class ClientHandler<path> implements Runnable {
*/ */
public String insertDataSQLBatch(String tableName, List<String> tableColumns, List<List<String>> dataList) { public String insertDataSQLBatch(String tableName, List<String> tableColumns, List<List<String>> dataList) {
StringBuilder sqlBuilder = new StringBuilder(); StringBuilder sqlBuilder = new StringBuilder();
StringBuilder batchValues = new StringBuilder();
for (int i = 0; i < dataList.size(); i++) { for (int i = 0; i < dataList.size(); i++) {
List<String> rowData = dataList.get(i); List<String> rowData = dataList.get(i);
if (!rowData.isEmpty()) { if (!rowData.isEmpty()) {
batchValues.append("(");
//默认ID和时间 //默认ID和时间
String values = "'"+UUID.randomUUID()+"','"+DateUtils.getDateNowString()+"'"; String itemValues = "'" + UUID.randomUUID() + "','" + DateUtils.getDateNowString() + "'";
for (int j = 0; j < rowData.size() - 1; j++) { for (int j = 0; j < rowData.size(); j++) {
Object value = rowData.get(j); Object value = rowData.get(j);
if (value instanceof Integer || value instanceof Long) { if (value instanceof Integer || value instanceof Long) {
values += value + ","; itemValues += "," + value;
} else { } else {
values += "'" + value + "',"; itemValues += ",'" + value + "'";
} }
} }
if (tableColumns.size() > rowData.size()) { if (tableColumns.size() > rowData.size()) {
for (int t = 0; t < tableColumns.size() - rowData.size(); t++) { for (int t = 0; t < tableColumns.size() - rowData.size(); t++) {
values += "''"; itemValues += ",''";
} }
} }
sqlBuilder.append("INSERT INTO "+tableName+" VALUES (" + values + ");\n"); batchValues.append(itemValues);
batchValues.append(")");
if (i < dataList.size() - 1) {
batchValues.append(",");
}
} }
} }
if(sqlBuilder !=null){ sqlBuilder.append("INSERT INTO " + tableName + " VALUES " + batchValues.toString() + ";\n");
if (sqlBuilder != null) {
return sqlBuilder.toString(); return sqlBuilder.toString();
} }
// String sql = "INSERT INTO " + tableName + " VALUES ";
// if (dataList.size() > 0) {
// for (int i = 0; i < dataList.size(); i++) {
// List<String> rowdatas = dataList.get(i);
// if (rowdatas.size() > 0) {
// sql += "(";
// sql += "\"" + UUID.randomUUID() + "\",";
// sql += "\"" + DateUtils.getDateNowString() + "\"";
// for (int n = 0; n < rowdatas.size(); n++) {
// sql += ",\"" + rowdatas.get(n) + "\"";
// }
// if (tableColumns.size() > rowdatas.size()) {
// for (int t = 0; t < tableColumns.size() - rowdatas.size(); t++) {
// sql += ",\"\"";
// }
// }
// sql += "),";
// }
//
// }
// }
// if (sql.lastIndexOf(",") > -1) {
// return sql.substring(0, sql.length() - 1);
// }
return null; return null;
} }
...@@ -605,9 +594,6 @@ public class ClientHandler<path> implements Runnable { ...@@ -605,9 +594,6 @@ public class ClientHandler<path> implements Runnable {
return header; return header;
} }
public static void main(String[] args) throws Exception {
requestInfoToSocketServer();
}
public void findDelimiter(InputStream is, String delimiter) throws IOException { public void findDelimiter(InputStream is, String delimiter) throws IOException {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
...@@ -682,6 +668,25 @@ public class ClientHandler<path> implements Runnable { ...@@ -682,6 +668,25 @@ public class ClientHandler<path> implements Runnable {
return new FileInputStream(fileTmpPath); return new FileInputStream(fileTmpPath);
} }
/**
* 按照指定数将一个list分割为多个list
*
* @param collection 源list
* @param size 按多少分割
* @param <T>
* @return
*/
// private static <T> List<List<T>> splitListBySize(Collection<T> collection, int size){
// if(size < 0){
// return null;
// }
// List<T> list = toArrayList(collection);
// // 计算可以拆分成几个list
// int count = (int) Math.ceil(list.size() / (double) size);
// // 使用流进行拆分
// return Stream.iterate(0,n->n+1).limit(count).map(i->list.stream().skip((long)i*size).limit(size).collect(Collectors.toList())).collect(Collectors.toList());
// }
//截取文件流 //截取文件流
public static InputStream intercept(InputStream is) throws IOException { public static InputStream intercept(InputStream is) throws IOException {
...@@ -749,6 +754,7 @@ public class ClientHandler<path> implements Runnable { ...@@ -749,6 +754,7 @@ public class ClientHandler<path> implements Runnable {
} }
private static void requestInfoToSocketServer() { private static void requestInfoToSocketServer() {
try { try {
Socket socket = new Socket("127.0.0.1", 7777); Socket socket = new Socket("127.0.0.1", 7777);
...@@ -769,4 +775,8 @@ public class ClientHandler<path> implements Runnable { ...@@ -769,4 +775,8 @@ public class ClientHandler<path> implements Runnable {
log.info("Socket传输数据异常!" + e.getMessage()); log.info("Socket传输数据异常!" + e.getMessage());
} }
} }
public static void main(String[] args) throws Exception {
requestInfoToSocketServer();
}
} }
\ No newline at end of file
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/config/SocketConfig.java
View file @ a7bb0ad7
...@@ -2,7 +2,9 @@ package com.yeejoin.amos.kgd.config; ...@@ -2,7 +2,9 @@ package com.yeejoin.amos.kgd.config;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
...@@ -29,6 +31,15 @@ public class SocketConfig { ...@@ -29,6 +31,15 @@ public class SocketConfig {
@Value("${amos.system.maas.url}") @Value("${amos.system.maas.url}")
private String hostAndPort; private String hostAndPort;
@Value("${spring.datasource.url}")
private String url;
@Value("${spring.datasource.username}")
private String username;
@Value("${spring.datasource.password}")
private String password;
private static final ThreadPoolExecutor threadpool = new ThreadPoolExecutor(15, 15, private static final ThreadPoolExecutor threadpool = new ThreadPoolExecutor(15, 15,
10L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); 10L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
...@@ -49,7 +60,7 @@ public class SocketConfig { ...@@ -49,7 +60,7 @@ public class SocketConfig {
clientSocket.setSoTimeout(10000); clientSocket.setSoTimeout(10000);
// 创建新线程处理连接 // 创建新线程处理连接
log.info("接收到客户端socket: {}", clientSocket.getRemoteSocketAddress()); log.info("接收到客户端socket: {}", clientSocket.getRemoteSocketAddress());
threadpool.execute(new ClientHandler(clientSocket,hostAndPort)); threadpool.execute(new ClientHandler(clientSocket, hostAndPort, url, username, password));
} }
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment