Commit ff779d37 authored by liguofu@yeejoin.com's avatar liguofu@yeejoin.com

优化:【空工大】解析Excel文件,自动创建数据库表和插入数据,并生成数据源

parent 5aff8f79
......@@ -3,11 +3,12 @@ package com.yeejoin.amos.kgd.config;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.util.StringUtil;
import com.yeejoin.amos.boot.biz.common.utils.DateUtils;
import com.yeejoin.amos.component.robot.AmosRequestContext;
import com.yeejoin.amos.kgd.message.Constant;
import com.yeejoin.amos.kgd.message.model.TableFieldModel;
import com.yeejoin.amos.kgd.message.model.TableModel;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -15,7 +16,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
......@@ -41,9 +41,6 @@ import java.util.regex.Pattern;
public class ClientHandler<path> implements Runnable {
private static final Logger log = LoggerFactory.getLogger(ClientHandler.class);
@Autowired
private JdbcTemplate jdbcTemplate;
private final Socket socket;
private final String hostAndPort;
......@@ -54,15 +51,14 @@ public class ClientHandler<path> implements Runnable {
}
@Value("${spring.datasource.url}")
private static String url;
private static String url="jdbc:mysql://172.16.3.101:3306/jd_bearing?allowMultiQueries=true&serverTimezone=GMT%2B8&characterEncoding=utf8";
@Value("${spring.datasource.username}")
private static String username;
private static String username="root";
@Value("${spring.datasource.password}")
private static String password;
private static String password="Yeejoin@2020";
public static final String DRIVER_CLASS_NAME = "";
public static final String DATABASE_NAME = "jd_bearing";
/*String*/
public static final String TYPE_STRING = "String";
......@@ -83,18 +79,12 @@ public class ClientHandler<path> implements Runnable {
try {
// 获取输入流和输出流
InputStream ips = socket.getInputStream();
ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream();
byte[] buffer2 = new byte[20];
int len2;
while ((len2 = ips.read(buffer2)) != -1){
outputStream2.write(buffer2,0,len2);
}
System.out.println(outputStream2.toString());
//存入缓存,方便重复使用
InputStreamCacher inputStreamCacher = new InputStreamCacher(ips);
// ips中,不是单纯的文件流,还有一些其他字段,所以要将单纯文件的部分过滤出来
// 获取文件地址信息
String filePath = filenameFilter(ips);
String filePath = filenameFilter(inputStreamCacher.getInputStream());
//截取文件名
String filenameWithSuffix = FileSystems.getDefault().getPath(filePath).getFileName().toString();
......@@ -115,27 +105,25 @@ public class ClientHandler<path> implements Runnable {
throw new Exception("导入失败,文件名称格式不正确!");
}
// ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream();
// byte[] buffer2 = new byte[20];
// int len2;
// while ((len2 = ips.read(buffer2)) != -1){
// outputStream2.write(buffer2,0,len2);
// }
// System.out.println(outputStream2.toString());
InputStream contentIps = fileInputStreamFilter(ips);
InputStream ipsClone2 = inputStreamCacher.getInputStream();
InputStream contentIps = fileInputStreamFilter(ipsClone2, postfix);
// 获取Excel表头内容
List<String> tableColumns = ExcelTool.readColumnsFromExcel(contentIps, filenameWithSuffix);
// 获取Excel表数据内容
List<List<String>> tableDatas = ExcelTool.readDataFromExcel(contentIps, filenameWithSuffix);
// 获取Excel表头内容 和 Excel表数据内容
Map<String, Object> tableInfoMap = ExcelTool.readFromExcel(contentIps, filenameWithSuffix);
// 新增mysql表结构和数据和数据源
if(StringUtil.isNotEmpty(tableName) && tableColumns != null && tableDatas != null){
this.addMySqlDatasources(tableColumns, tableDatas, hostAndPort, filename);
if(StringUtil.isNotEmpty(tableName) && !tableInfoMap.isEmpty() && tableInfoMap.get("tableColumns") != null){
List<String> tableColumns = (List<String>) tableInfoMap.get("tableColumns");
List<List<String>> tableDatas = (List<List<String>>) tableInfoMap.get("tableDatas");
this.addMySqlDatasources(tableColumns, tableDatas, hostAndPort, tableName);
}
// 关闭连接
socket.close();
ips.close();
// 销毁缓存
if(inputStreamCacher != null) inputStreamCacher.destroyCache();
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
......@@ -143,15 +131,16 @@ public class ClientHandler<path> implements Runnable {
}
}
//新增My Sql数据源
private void addMySqlDatasources(List<String> tableColumns, List<List<String>> DataList, String hostAndPort, String tableName) throws Exception {
private void addMySqlDatasources(List<String> tableColumns, List<List<String>> dataList, String hostAndPort, String tableName) throws Exception {
AmosRequestContext robotAuthentication = SpringContextHelper.getBean(AmosRequestContext.class);
if (Objects.nonNull(robotAuthentication)) {
// String token = robotAuthentication.getToken();
// String product = robotAuthentication.getProduct();
// String appKey = robotAuthentication.getAppKey();
String token = "cad32761-9cc0-43e1-b1bf-4488d2e04942"; // 如果token、失效,就去浏览器复制最新的token、
String token = "ab2d46e6-243b-4ea5-8f2f-126e5b2d87ae"; // 如果token、失效,就去浏览器复制最新的token、
String product = "AMOS_STUDIO_WEB";
String appKey = "AMOS_STUDIO";
......@@ -169,7 +158,7 @@ public class ClientHandler<path> implements Runnable {
state = (Statement) connMap.get("statement");
//2.判断数据库表是否存在【不存在时,新增表。存在时,直接录入数据.】
// 获取数据库表名
ResultSet tables = conn.getMetaData().getTables(null, null, DATABASE_NAME, null);
ResultSet tables = conn.getMetaData().getTables(null, null, tableName, null);
// 2.1是否存在表
if (!tables.next()) {
// 不存在则创建表
......@@ -177,15 +166,7 @@ public class ClientHandler<path> implements Runnable {
List<TableFieldModel> tableFieldList = new ArrayList<>();
tableModel.setTableName(tableName);
if (tableColumns.size() > 0) {
for (int i = 0; i < tableColumns.size(); i++) {
TableFieldModel tableFieldModel = new TableFieldModel();
tableFieldModel.setFieldName(tableColumns.get(i));
tableFieldModel.setFieldType(TYPE_STRING);
tableFieldModel.setFieldLength("255");
tableFieldList.add(tableFieldModel);
}
}
// start 添加默认字段ID和createDate
TableFieldModel defaultFieldId = new TableFieldModel();
defaultFieldId.setFieldType(TYPE_STRING);
......@@ -194,14 +175,27 @@ public class ClientHandler<path> implements Runnable {
defaultFieldId.setIsNull("0");
defaultFieldId.setFieldLength("50");
tableFieldList.add(defaultFieldId);
TableFieldModel defaultFieldCreateDate = new TableFieldModel();
defaultFieldCreateDate.setFieldType(TYPE_DATETIME);
defaultFieldCreateDate.setFieldType(TYPE_STRING);
defaultFieldCreateDate.setFieldName("CREATEDATE");
defaultFieldCreateDate.setIsNull("0");
tableFieldList.add(defaultFieldId);
defaultFieldCreateDate.setFieldLength("50");
tableFieldList.add(defaultFieldCreateDate);
// end 添加默认字段ID和createDate
//Excel中字段
if (tableColumns.size() > 0) {
for (int i = 0; i < tableColumns.size(); i++) {
TableFieldModel tableFieldModel = new TableFieldModel();
tableFieldModel.setFieldName(tableColumns.get(i));
tableFieldModel.setFieldType(TYPE_STRING);
tableFieldModel.setFieldLength("255");
tableFieldList.add(tableFieldModel);
}
}
tableModel.setTableFields(tableFieldList);
//生成建表语句
String createTableSql = createTableSQL(tableModel);
if (StringUtil.isEmpty(createTableSql)) {
throw new Exception("mysql建表语句出错!");
......@@ -209,6 +203,12 @@ public class ClientHandler<path> implements Runnable {
// 创建表
state.executeUpdate(createTableSql);
}
// 2.2插入数据
String insertSQL = insertDataSQLBatch(tableName, tableColumns, dataList);
if(StringUtil.isNotEmpty(insertSQL)){
state.execute(insertSQL);
}
}catch (Exception e){
e.printStackTrace();
throw new Exception("数据表创建失败!");
......@@ -217,8 +217,6 @@ public class ClientHandler<path> implements Runnable {
state.close();
conn.close();
}
// 2.2插入数据
insertDataSQL(tableName, tableColumns, DataList);
//3.新增mass数据源
String connsUrl = "http://" + hostAndPort + "/maas/dsm/datasources";
......@@ -244,8 +242,6 @@ public class ClientHandler<path> implements Runnable {
Statement statement = null;
Map<String, Object> map = new HashMap<>();
try {
// 连接数据库
// Class.forName(DRIVER_CLASS_NAME);
// 获取数据库连接
connection = DriverManager.getConnection(url, username, password);
// 根据连接获取可执行Statement
......@@ -267,11 +263,13 @@ public class ClientHandler<path> implements Runnable {
* @return
*/
public static String createTableSQL(TableModel tableModel){
String tableName = tableModel.getTableName();
// 创建主键集合
List<String> priKeyList = new ArrayList<String>();
// 创建 StringBuffer 拼接sql
StringBuffer sb = new StringBuffer();
sb.append("CREATE TABLE `"+ tableModel.getTableName() +"` (\n");
sb.append("CREATE TABLE `"+ tableName.toUpperCase() +"` (\n");
List<TableFieldModel> tableFields = tableModel.getTableFields();
for (int i = 0; i < tableFields.size(); i++) {
// 当前条数据
......@@ -280,12 +278,14 @@ public class ClientHandler<path> implements Runnable {
String fieldType = judgeDataType(field.getFieldType());
sb.append(""+ field.getFieldName() +"");
if ("double".equals(fieldType)){
// 特殊处理 `age` double(23,0) DEFAULT NULL COMMENT '年龄',
// 追加列
sb.append(" "+fieldType+"("+field.getFieldLength()+","+ field.getDecimalPoint() +") ");
}else if ("decimal".equals(fieldType)){
// 追加列
sb.append(" "+fieldType+"("+field.getFieldLength()+","+ field.getDecimalPoint() +") ");
}else if("datetime".equals(fieldType)){
// 追加列
sb.append(" "+fieldType+" ");
}else {
// 追加列
sb.append(" "+fieldType+"("+field.getFieldLength()+") ");
......@@ -358,27 +358,72 @@ public class ClientHandler<path> implements Runnable {
}
return sb.toString();
}
/**
* 创建插入数据的SQL
* 创建单条插入数据的SQL
* @param tableName 表名
* @param tableColumns 列名
* @param DataList 数据
* @param dataList 数据
* @return
*/
public void insertDataSQL(String tableName, List<String> tableColumns, List<List<String>> DataList){
public String insertDataSQL(String tableName, List<String> tableColumns, List<String> dataList){
String sql = "INSERT INTO "+tableName+" VALUES ";
if(DataList.size()>0){
for (int i=0; i<DataList.size(); i++){
sql+= "("+DataList+"),";
if(dataList.size()>0){
sql+= "(";
sql += "\""+ UUID.randomUUID() +"\",";
sql += "\""+ DateUtils.getDateNowString() +"\"";
for (int n = 0; n < dataList.size(); n++){
sql += ",\""+dataList.get(n)+"\"";
}
if(tableColumns.size() > dataList.size()){
for (int t = 0; t < tableColumns.size() - dataList.size(); t++){
sql += ",\"\"";
}
}
sql+="),";
}
if(sql.lastIndexOf(",")>-1){
return sql.substring(0, sql.length() - 1);
}
return null;
}
/**da
* 创建批量插入数据的SQL
* @param tableName 表名
* @param tableColumns 列名
* @param dataList 数据
* @return
*/
public String insertDataSQLBatch(String tableName, List<String> tableColumns, List<List<String>> dataList){
String sql = "INSERT INTO "+tableName+" VALUES ";
if(dataList.size()>0){
for (int i = 0; i < dataList.size(); i++){
List<String> rowdatas = dataList.get(i);
if(rowdatas.size()>0){
sql+= "(";
sql += "\""+ UUID.randomUUID() +"\",";
sql += "\""+ DateUtils.getDateNowString() +"\"";
for (int n = 0; n < rowdatas.size(); n++){
sql += ",\""+rowdatas.get(n)+"\"";
}
if(tableColumns.size() > rowdatas.size()){
for (int t = 0; t < tableColumns.size() - rowdatas.size(); t++){
sql += ",\"\"";
}
}
sql+="),";
}
}
}
if(sql.lastIndexOf(",")>-1){
sql = sql.substring(0,sql.lastIndexOf(","));
jdbcTemplate.execute(sql);
return sql.substring(0, sql.length() - 1);
}
return null;
}
/**jdbcTemplate
/**
* 判断类型
* varchar
* int
......@@ -550,20 +595,12 @@ public class ClientHandler<path> implements Runnable {
}
public void findDelimiter(InputStream is, String delimiter) throws IOException {
// StringBuilder sb = new StringBuilder();
// int nextByte;
// while ((nextByte = is.read()) != -1) {
// char c = (char) nextByte;
// sb.append(c);
// if (sb.toString().contains(delimiter)) {
// return;
// }
// }
// 将输入流包装成 BufferedReader
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(is));
String line;
while ((line = bufferedReader.readLine()) != null) {
if (StringUtils.isNotEmpty(line) && line.contains(delimiter)) {
StringBuilder sb = new StringBuilder();
int nextByte;
while ((nextByte = is.read()) != -1) {
char c = (char) nextByte;
sb.append(c);
if (sb.toString().contains(delimiter)) {
return;
}
}
......@@ -588,23 +625,6 @@ public class ClientHandler<path> implements Runnable {
return buf;
}
private InputStream getDelimiterContent(InputStream is, String start, String end) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
StringBuilder contentBuilder = new StringBuilder();
boolean isInsideTitleSection = false;
String line;
while ((line = reader.readLine()) != null) {
if (!isInsideTitleSection && line.contains(start)) {
isInsideTitleSection = true;
} else if (isInsideTitleSection && line.contains(end)) {
break;
} else if (isInsideTitleSection) {
contentBuilder.append(line).append("\n");
}
}
return new ByteArrayInputStream(contentBuilder.toString().getBytes("UTF-8"));
}
public String filenameFilter(InputStream is) throws IOException {
BufferedInputStream bis = new BufferedInputStream(is);
......@@ -615,48 +635,35 @@ public class ClientHandler<path> implements Runnable {
//校验内容格式
findDelimiter(bis, start);
//获取指定标记中的内容
InputStream resultContent = getDelimiterContent(is, start, end);
byte[] bytes = new byte[resultContent.available()];
int byteRead = resultContent.read(bytes);
if(byteRead != -1 ){
return new String(bytes);
byte[] endDelimiter = end.getBytes();
int byteRead;
byte[] buf = new byte[4096];
if ((byteRead = bis.read(buf)) != -1) {
byte[] newByte = checkDelimiter(buf, byteRead, endDelimiter);
return new String(newByte, "GBK");
}
// byte[] endDelimiter = end.getBytes();
// int byteRead;
// byte[] buf = new byte[4096];
// if ((byteRead = bis.read(buf)) != -1) {
// byte[] newByte = checkDelimiter(buf, byteRead, endDelimiter);
// return new String(newByte, "GBK");
// }
throw new IOException("没有输入流内容或者文件名太长");
}
public InputStream fileInputStreamFilter(InputStream is) throws IOException {
public InputStream fileInputStreamFilter(InputStream is, String postfix) throws IOException {
BufferedInputStream bis = new BufferedInputStream(is);
// String fileTmpPath = "D:\\tmp.xlsx";
// FileOutputStream fos = new FileOutputStream(fileTmpPath);
String fileTmpPath = "xls".equals(postfix) ? "D:\\tmp.xls" : "D:\\tmp.xlsx";
FileOutputStream fos = new FileOutputStream(fileTmpPath);
String start = "##STATITLE##";
String end = "##ENDTITLE##";
String start = "##STAFILE##";
String end = "##ENDFILE##";
//校验内容格式
findDelimiter(bis, start);
//获取指定标记中的内容
InputStream resultContent = getDelimiterContent(is, start, end);
return resultContent;
// byte[] endDelimiter = end.getBytes();
// int byteRead;
// byte[] buf = new byte[4000];
// while ((byteRead = bis.read(buf)) != -1) {
// byte[] newByte = checkDelimiter(buf, byteRead, endDelimiter);
// fos.write(newByte, 0, Math.min(byteRead, newByte.length));
//
// }
// return new FileInputStream(fileTmpPath);
byte[] endDelimiter = end.getBytes();
int byteRead;
byte[] buf = new byte[4000];
while ((byteRead = bis.read(buf)) != -1) {
byte[] newByte = checkDelimiter(buf, byteRead, endDelimiter);
fos.write(newByte, 0, Math.min(byteRead, newByte.length));
}
return new FileInputStream(fileTmpPath);
}
//截取文件流
......@@ -731,7 +738,7 @@ public class ClientHandler<path> implements Runnable {
Socket socket = new Socket("127.0.0.1", 7777);
// Socket socket = new Socket("localhost", 7777);
OutputStream ops = socket.getOutputStream();
FileInputStream fis = new FileInputStream("D:\\SamData\\RecordXLS\\測試\\1.xlsx");
FileInputStream fis = new FileInputStream("D:\\SamData\\RecordXLS\\測試\\DYKZZH2023年04月13日14时49分36秒.xls");
int len = 0;
byte[] bs = new byte[20480];
while ((len = fis.read(bs)) != -1) {
......
......@@ -7,9 +7,6 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.web.multipart.MultipartFile;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.NumberFormat;
......@@ -46,12 +43,13 @@ public class ExcelTool {
}
/**
* 解析Excel表头
* 解析Excel
*
* @param inputStream 文件
* @return
*/
public static List<String> readColumnsFromExcel(InputStream inputStream, String filename) throws Exception {
public static Map<String, Object> readFromExcel(InputStream inputStream, String filename) throws Exception {
Map<String, Object> resultTableMap = new HashMap<>();
POIFSFileSystem pfs = null;
Workbook workbook = null;
try {
......@@ -70,54 +68,32 @@ public class ExcelTool {
if (Objects.isNull(sheet)) {
return null;
}
List<String> titleList = new LinkedList<>();
numberFormat = NumberFormat.getNumberInstance();
// 创建一个集合根据下标来确定每个单元格对应对象的什么属性
List<List<String>> dataList = new ArrayList<>();
// 获取第一行数据(假如第一行就是列名)
Row sheetTitleRow = sheet.getRow(sheet.getFirstRowNum());
// 取出最后一列
short lastCellNum = sheetTitleRow.getLastCellNum();
List<String> sheetTitleList = new LinkedList<>();
for (int i = 0; i < lastCellNum; i++) {
// 取出每一列的名
String cellValue = sheetTitleRow.getCell(i).getStringCellValue();
sheetTitleList.add(cellValue);
titleList.add(cellValue);
}
return sheetTitleList;
}
/**
* 解析Excel数据
*
* @param inputStream 文件
* @return
*/
public static List<List<String>> readDataFromExcel(InputStream inputStream, String filename) throws Exception {
POIFSFileSystem pfs = null;
Workbook workbook = null;
try {
// 解析xls和xlsx不兼容问题
workbook = getWorkBook(pfs, workbook, inputStream, filename);
} catch (IOException e) {
e.printStackTrace();
throw new Exception("模板保存异常。");
}
if (workbook == null) {
throw new Exception("请使用模板上传文件");
}
numberFormat = NumberFormat.getNumberInstance();
// 创建一个集合根据下标来确定每个单元格对应对象的什么属性
List<List<String>> resultList = new ArrayList<>();
resultTableMap.put("tableColumns", titleList);
// 获取表格第一个sheet的内容
Sheet sheetAt = workbook.getSheetAt(0);
// 获得sheet总行数
int lastRowNum = sheetAt.getLastRowNum();
int lastRowNum = sheet.getLastRowNum();
if (lastRowNum < 1) {
throw new Exception("数据错误");
}
// 开始读取,不读取表头所以从第二行开始
for (int i = 1; i <= lastRowNum; i++) {
// 获取每一行
Row row = sheetAt.getRow(i);
Row row = sheet.getRow(i);
// 行为空不读取
if (row == null) {
continue;
......@@ -129,8 +105,8 @@ public class ExcelTool {
}
List<String> rowList = new ArrayList<>();
//添加数据
short lastCellNum = row.getLastCellNum();
for (int j = 0; j < lastCellNum; j++) {
short lastCellNum2 = row.getLastCellNum();
for (int j = 0; j < lastCellNum2; j++) {
Cell cellOne = row.getCell(j);
try {
String item = convertData(cellOne);
......@@ -140,16 +116,17 @@ public class ExcelTool {
System.out.println(i + "行" + j + "列数据转换出现异常");
rowList.add("");
}
resultList.add(rowList);
}
dataList.add(rowList);
//规避行数数据后几行为空
if (rowList.size() < lastCellNum) {
if (rowList.size() < lastCellNum2) {
for (int k = 0; k < 15 - rowList.size(); k++) {
rowList.add("");
}
}
}
return resultList;
resultTableMap.put("tableDatas", dataList);
return resultTableMap;
}
/**
......
package com.yeejoin.amos.kgd.config;
import org.apache.commons.lang3.ObjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
public class InputStreamCacher {
private static final Logger log = LoggerFactory.getLogger(InputStreamCacher.class);
/**
* 将InputStream中的字节保存到ByteArrayOutputStream中
*/
private ByteArrayOutputStream byteArrayOutputStream;
private InputStream inputStream;
public InputStreamCacher(InputStream inputStream){
if(inputStream == null){
return;
}
this.inputStream = inputStream;
initCache();
}
/**
* 初始化
*/
private void initCache(){
byteArrayOutputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int len;
try{
while ((len = inputStream.read(buffer)) >-1 ){
byteArrayOutputStream.write(buffer, 0 , len);
}
byteArrayOutputStream.flush();
}catch (IOException e){
log.error(e.getMessage(), e);
}
}
/**
* 获取缓存流
* @return InputStream
*/
public InputStream getInputStream(){
if(byteArrayOutputStream == null){
return this.inputStream;
}
return new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
}
/**
* 销毁
*/
public void destroyCache(){
this.byteArrayOutputStream = null;
if(this.inputStream != null){
try {
this.inputStream.close();
}catch (IOException e){
e.printStackTrace();
}
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment