Skip to content

A
amos-boot-biz
Commit a7bb0ad7 authored by liguofu@yeejoin.com's avatar liguofu@yeejoin.com
Browse files
Options

优化：【空工大】解析Excel文件，大量数据插入时，分批插入

parent a58f9b8f
Show whitespace changes
Inline Side-by-side
Showing with 79 additions and 57 deletions
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/config/ClientHandler.java
View file @ a7bb0ad7
......@@ -8,11 +8,8 @@ import com.yeejoin.amos.component.robot.AmosRequestContext;
import com.yeejoin.amos.kgd.message.Constant;
import com.yeejoin.amos.kgd.message.model.TableFieldModel;
import com.yeejoin.amos.kgd.message.model.TableModel;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
......@@ -31,6 +28,8 @@ import java.sql.Statement;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* @Author: xl
......@@ -45,19 +44,19 @@ public class ClientHandler<path> implements Runnable {
private final String hostAndPort;
public ClientHandler(Socket socket, String hostAndPort) {
this.socket = socket;
this.hostAndPort = hostAndPort; //127.0.0.1:30009
}
private final String url;
@Value("${spring.datasource.url}")
private static String url = "jdbc:mysql://172.16.3.101:3306/jd_bearing?allowMultiQueries=true&serverTimezone=GMT%2B8&characterEncoding=utf8";
private final String username;
@Value("${spring.datasource.username}")
private static String username = "root";
private final String password;
@Value("${spring.datasource.password}")
private static String password = "Yeejoin@2020";
public ClientHandler(Socket socket, String hostAndPort, String url, String username, String password) {
this.socket = socket;
this.hostAndPort = hostAndPort; //127.0.0.1:30009
this.url = url; // 数据库连接地址
this.username = username;// 数据库用户名
this.password = password;// 数据库密码
}
public static final String DATABASE_NAME = "jd_bearing";
/*String*/
......@@ -205,14 +204,21 @@ public class ClientHandler<path> implements Runnable {
}
// 2.2插入数据
if (dataList.size() > 0) {
for (int i = 0; i < dataList.size(); i++) {
List<List<String>> lists;
if (dataList.size() > 1000 && i % 1000 == 0 && i > 0) {
lists = (i + 1000) < dataList.size() ? dataList.subList(i, i + 1000) : dataList.subList(i, dataList.size());
}else {
lists = dataList;
if (dataList.size() <= 1000) {
String insertSQL = insertDataSQLBatch(tableName, tableColumns, dataList);
if (StringUtil.isNotEmpty(insertSQL)) {
state.execute(insertSQL);
}
if(lists.size()>0){
} else {
for (int i = 0; i <= dataList.size(); i++) {
List<List<String>> lists = null;
if (i == dataList.size()) {
lists = dataList.subList(dataList.size() - i % 1000, dataList.size());
}
if (i % 1000 == 0 && i > 0) {
lists = dataList.subList(i - 1000, i);
}
if (lists != null) {
String insertSQL = insertDataSQLBatch(tableName, tableColumns, lists);
if (StringUtil.isNotEmpty(insertSQL)) {
state.execute(insertSQL);
......@@ -220,6 +226,7 @@ public class ClientHandler<path> implements Runnable {
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
......@@ -249,7 +256,7 @@ public class ClientHandler<path> implements Runnable {
/**
* 连接MySQL数据库
*/
public static Map<String, Object> connectMySQL() {
public Map<String, Object> connectMySQL() {
Connection connection = null;
Statement statement = null;
Map<String, Object> map = new HashMap<>();
......@@ -383,56 +390,38 @@ public class ClientHandler<path> implements Runnable {
*/
public String insertDataSQLBatch(String tableName, List<String> tableColumns, List<List<String>> dataList) {
StringBuilder sqlBuilder = new StringBuilder();
StringBuilder batchValues = new StringBuilder();
for (int i = 0; i < dataList.size(); i++) {
List<String> rowData = dataList.get(i);
if (!rowData.isEmpty()) {
batchValues.append("(");
//默认ID和时间
String values = "'"+UUID.randomUUID()+"','"+DateUtils.getDateNowString()+"'";
for (int j = 0; j < rowData.size() - 1; j++) {
String itemValues = "'" + UUID.randomUUID() + "','" + DateUtils.getDateNowString() + "'";
for (int j = 0; j < rowData.size(); j++) {
Object value = rowData.get(j);
if (value instanceof Integer || value instanceof Long) {
values += value + ",";
itemValues += "," + value;
} else {
values += "'" + value + "',";
itemValues += ",'" + value + "'";
}
}
if (tableColumns.size() > rowData.size()) {
for (int t = 0; t < tableColumns.size() - rowData.size(); t++) {
values += "''";
itemValues += ",''";
}
}
sqlBuilder.append("INSERT INTO "+tableName+" VALUES (" + values + ");\n");
batchValues.append(itemValues);
batchValues.append(")");
if (i < dataList.size() - 1) {
batchValues.append(",");
}
}
}
if(sqlBuilder !=null){
sqlBuilder.append("INSERT INTO " + tableName + " VALUES " + batchValues.toString() + ";\n");
if (sqlBuilder != null) {
return sqlBuilder.toString();
}
// String sql = "INSERT INTO " + tableName + " VALUES ";
// if (dataList.size() > 0) {
// for (int i = 0; i < dataList.size(); i++) {
// List<String> rowdatas = dataList.get(i);
// if (rowdatas.size() > 0) {
// sql += "(";
// sql += "\"" + UUID.randomUUID() + "\",";
// sql += "\"" + DateUtils.getDateNowString() + "\"";
// for (int n = 0; n < rowdatas.size(); n++) {
// sql += ",\"" + rowdatas.get(n) + "\"";
// }
// if (tableColumns.size() > rowdatas.size()) {
// for (int t = 0; t < tableColumns.size() - rowdatas.size(); t++) {
// sql += ",\"\"";
// }
// }
// sql += "),";
// }
//
// }
// }
// if (sql.lastIndexOf(",") > -1) {
// return sql.substring(0, sql.length() - 1);
// }
return null;
}
......@@ -605,9 +594,6 @@ public class ClientHandler<path> implements Runnable {
return header;
}
public static void main(String[] args) throws Exception {
requestInfoToSocketServer();
}
public void findDelimiter(InputStream is, String delimiter) throws IOException {
StringBuilder sb = new StringBuilder();
......@@ -682,6 +668,25 @@ public class ClientHandler<path> implements Runnable {
return new FileInputStream(fileTmpPath);
}
/**
* 按照指定数将一个list分割为多个list
*
* @param collection 源list
* @param size 按多少分割
* @param <T>
* @return
*/
// private static <T> List<List<T>> splitListBySize(Collection<T> collection, int size){
// if(size < 0){
// return null;
// }
// List<T> list = toArrayList(collection);
// // 计算可以拆分成几个list
// int count = (int) Math.ceil(list.size() / (double) size);
// // 使用流进行拆分
// return Stream.iterate(0,n->n+1).limit(count).map(i->list.stream().skip((long)i*size).limit(size).collect(Collectors.toList())).collect(Collectors.toList());
// }
//截取文件流
public static InputStream intercept(InputStream is) throws IOException {
......@@ -749,6 +754,7 @@ public class ClientHandler<path> implements Runnable {
}
private static void requestInfoToSocketServer() {
try {
Socket socket = new Socket("127.0.0.1", 7777);
......@@ -769,4 +775,8 @@ public class ClientHandler<path> implements Runnable {
log.info("Socket传输数据异常!" + e.getMessage());
}
}
public static void main(String[] args) throws Exception {
requestInfoToSocketServer();
}
}
\ No newline at end of file
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/config/SocketConfig.java
View file @ a7bb0ad7
......@@ -2,7 +2,9 @@ package com.yeejoin.amos.kgd.config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
......@@ -29,6 +31,15 @@ public class SocketConfig {
@Value("${amos.system.maas.url}")
private String hostAndPort;
@Value("${spring.datasource.url}")
private String url;
@Value("${spring.datasource.username}")
private String username;
@Value("${spring.datasource.password}")
private String password;
private static final ThreadPoolExecutor threadpool = new ThreadPoolExecutor(15, 15,
10L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
......@@ -49,7 +60,7 @@ public class SocketConfig {
clientSocket.setSoTimeout(10000);
// 创建新线程处理连接
log.info("接收到客户端socket: {}", clientSocket.getRemoteSocketAddress());
threadpool.execute(new ClientHandler(clientSocket,hostAndPort));
threadpool.execute(new ClientHandler(clientSocket, hostAndPort, url, username, password));
}
} catch (IOException e) {
throw new RuntimeException(e);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment