Skip to content

A
amos-boot-biz
Commit cae4ba2e authored by liguofu@yeejoin.com's avatar liguofu@yeejoin.com
Browse files
Options

修改：【空工大】解析Excel文件，自动创建数据库表和插入数据，并生成数据源

parent 42753a79
Hide whitespace changes
Inline Side-by-side
Showing with 577 additions and 11 deletions
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/config/ClientHandler.java
View file @ cae4ba2e
......@@ -2,13 +2,21 @@ package com.yeejoin.amos.kgd.config;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.util.StringUtil;
import com.netflix.discovery.shared.transport.decorator.EurekaHttpClientDecorator;
import com.yeejoin.amos.component.robot.AmosRequestContext;
import com.yeejoin.amos.kgd.message.Constant;
import com.yeejoin.amos.kgd.message.model.TableFieldModel;
import com.yeejoin.amos.kgd.message.model.TableModel;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
......@@ -18,8 +26,14 @@ import java.io.*;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @Author: xl
......@@ -30,6 +44,9 @@ import java.util.*;
public class ClientHandler<path> implements Runnable {
private static final Logger log = LoggerFactory.getLogger(ClientHandler.class);
@Autowired
private JdbcTemplate jdbcTemplate;
private final Socket socket;
private final String hostAndPort;
......@@ -39,29 +56,364 @@ public class ClientHandler<path> implements Runnable {
this.hostAndPort = hostAndPort; //127.0.0.1:30009
}
@Value("${spring.datasource.url}")
private static String url;
@Value("${spring.datasource.username}")
private static String username;
@Value("${spring.datasource.password}")
private static String password;
public static final String DRIVER_CLASS_NAME = "";
public static final String DATABASE_NAME = "jd_bearing";
/*String*/
public static final String TYPE_STRING = "String";
/*Integer*/
public static final String TYPE_INTEGER = "Integer";
/*Double*/
public static final String TYPE_DOUBLE = "Double";
/*Date*/
public static final String TYPE_DATETIME = "DateTime";
/*BigDecimal*/
public static final String TYPE_BIGDECIMAL = "BigDecimal";
/*text*/
public static final String TYPE_TEXT = "text";
@Override
public void run() {
try {
// 获取输入流和输出流
InputStream ips = socket.getInputStream();
ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream();
byte[] buffer2 = new byte[20];
int len2;
while ((len2 = ips.read(buffer2)) != -1){
outputStream2.write(buffer2,0,len2);
}
System.out.println(outputStream2.toString());
// ips中，不是单纯的文件流，还有一些其他字段，所以要将单纯文件的部分过滤出来
// 获取文件地址信息
String filePath = filenameFilter(ips);
//截取文件名
String filenameWithSuffix = FileSystems.getDefault().getPath(filePath).getFileName().toString();
String postfix = filenameWithSuffix.substring(filenameWithSuffix.lastIndexOf(".") + 1);
if (!"xlsx".equals(postfix) && !"xls".equals(postfix)) {
throw new Exception("导入失败，请选择正确的文件格式支持xlsx或xls");
}
// 获取文件名
String filename = filenameWithSuffix.substring(0, filenameWithSuffix.lastIndexOf("."));
// 获取表名称
String tableName = "";
// 匹配开头到日期时间开始前的字符
Pattern pattern = Pattern.compile("^.*?(?=\\d{4}年\\d{1,2}月\\d{1,2}日\\d{2}时\\d{2}分\\d{2}秒)");
Matcher matcher = pattern.matcher(filename);
if (matcher.find()) {
tableName = matcher.group();
} else {
throw new Exception("导入失败，文件名称格式不正确！");
}
// ByteArrayOutputStream outputStream2 = new ByteArrayOutputStream();
// byte[] buffer2 = new byte[20];
// int len2;
// while ((len2 = ips.read(buffer2)) != -1){
// outputStream2.write(buffer2,0,len2);
// }
// System.out.println(outputStream2.toString());
ips = fileInputStreamFilter(ips);
// 处理客户端请求
log.info("收到客户端数据: ");
String path = this.upload2Maas(ips, hostAndPort, filename);
log.info("调用maas服务返回结果: {}", path);
// 获取Excel表头内容
List<String> tableColumns = ExcelTool.readColumnsFromExcel(ips, filenameWithSuffix);
// 获取Excel表数据内容
List<List<String>> tableDatas = ExcelTool.readDataFromExcel(ips, filenameWithSuffix);
// 新增mysql表结构和数据和数据源
if(StringUtil.isNotEmpty(tableName) && tableColumns != null && tableDatas != null){
this.addMySqlDatasources(tableColumns, tableDatas, hostAndPort, filename);
}
// 关闭连接
byte[] bytes = path.getBytes();
socket.close();
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
//新增My Sql数据源
private void addMySqlDatasources(List<String> tableColumns, List<List<String>> DataList, String hostAndPort, String tableName) throws Exception {
AmosRequestContext robotAuthentication = SpringContextHelper.getBean(AmosRequestContext.class);
if (Objects.nonNull(robotAuthentication)) {
// String token = robotAuthentication.getToken();
// String product = robotAuthentication.getProduct();
// String appKey = robotAuthentication.getAppKey();
String token = "cad32761-9cc0-43e1-b1bf-4488d2e04942"; // 如果token、失效，就去浏览器复制最新的token、
String product = "AMOS_STUDIO_WEB";
String appKey = "AMOS_STUDIO";
Connection conn = null;
Statement state = null;
try {
//1.jdbc连接数据库
Map<String, Object> connMap = connectMySQL();
if (connMap.isEmpty()) {
throw new Exception("获取数据库连接失败...");
}
// 获取数据库连接
conn = (Connection) connMap.get("connection");
// 获取SQL执行环境
state = (Statement) connMap.get("statement");
//2.判断数据库表是否存在【不存在时，新增表。存在时，直接录入数据.】
// 获取数据库表名
ResultSet tables = conn.getMetaData().getTables(null, null, DATABASE_NAME, null);
// 2.1是否存在表
if (!tables.next()) {
// 不存在则创建表
TableModel tableModel = new TableModel();
List<TableFieldModel> tableFieldList = new ArrayList<>();
tableModel.setTableName(tableName);
if (tableColumns.size() > 0) {
for (int i = 0; i < tableColumns.size(); i++) {
TableFieldModel tableFieldModel = new TableFieldModel();
tableFieldModel.setFieldName(tableColumns.get(i));
tableFieldModel.setFieldType(TYPE_STRING);
tableFieldModel.setFieldLength("255");
tableFieldList.add(tableFieldModel);
}
}
// start 添加默认字段ID和createDate
TableFieldModel defaultFieldId = new TableFieldModel();
defaultFieldId.setFieldType(TYPE_STRING);
defaultFieldId.setFieldName("ID");
defaultFieldId.setPrimaryKey("1");
defaultFieldId.setIsNull("0");
defaultFieldId.setFieldLength("50");
tableFieldList.add(defaultFieldId);
TableFieldModel defaultFieldCreateDate = new TableFieldModel();
defaultFieldCreateDate.setFieldType(TYPE_DATETIME);
defaultFieldCreateDate.setFieldName("CREATEDATE");
defaultFieldCreateDate.setIsNull("0");
tableFieldList.add(defaultFieldId);
// end 添加默认字段ID和createDate
tableModel.setTableFields(tableFieldList);
String createTableSql = createTableSQL(tableModel);
if (StringUtil.isEmpty(createTableSql)) {
throw new Exception("mysql建表语句出错！");
}
// 创建表
state.executeUpdate(createTableSql);
}
}catch (Exception e){
e.printStackTrace();
throw new Exception("数据表创建失败！");
}finally {
// 释放资源
state.close();
conn.close();
}
// 2.2插入数据
insertDataSQL(tableName, tableColumns, DataList);
//3.新增mass数据源
String connsUrl = "http://" + hostAndPort + "/maas/dsm/datasources";
Map<String, String> params = new HashMap<>();
params.put("name", tableName);
params.put("subtype", "mysql");
params.put("type", "DataBase");
params.put("groupid", "6bc573d8-9534-4a93-8315-46be74f4c311");
params.put("detail", "{\"hostname\":\"172.16.3.101\",\"port\":3306,\"username\":\"" + username + "\",\"password\":\"" + password + "\",\"name\":\"" + tableName + "\",\"database\":\"" + DATABASE_NAME + "\",\"dbtype\":\"mysql\"}");
HttpEntity<Map<String, String>> requestEntity2 = new HttpEntity<>(params, getHeader(token, product, appKey, hostAndPort, false));
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> response = restTemplate.exchange(connsUrl, HttpMethod.POST, requestEntity2, String.class);
}
}
/**
* 连接MySQL数据库
*/
public static Map<String, Object> connectMySQL(){
Connection connection = null;
Statement statement = null;
Map<String, Object> map = new HashMap<>();
try {
// 连接数据库
// Class.forName(DRIVER_CLASS_NAME);
// 获取数据库连接
connection = DriverManager.getConnection(url, username, password);
// 根据连接获取可执行Statement
statement = connection.createStatement();
map.put("connection",connection);
map.put("statement",statement);
return map;
}catch (Exception e){
e.printStackTrace();
log.error("连接数据库错误！");
}
return null;
}
/**
* 创建生成表的SQL
* @param tableModel
* @return
*/
public static String createTableSQL(TableModel tableModel){
// 创建主键集合
List<String> priKeyList = new ArrayList<String>();
// 创建 StringBuffer 拼接sql
StringBuffer sb = new StringBuffer();
sb.append("CREATE TABLE `"+ tableModel.getTableName() +"` (\n");
List<TableFieldModel> tableFields = tableModel.getTableFields();
for (int i = 0; i < tableFields.size(); i++) {
// 当前条数据
TableFieldModel field = tableFields.get(i);
// 判断数据类型
String fieldType = judgeDataType(field.getFieldType());
sb.append(""+ field.getFieldName() +"");
if ("double".equals(fieldType)){
// 特殊处理 `age` double(23,0) DEFAULT NULL COMMENT '年龄',
// 追加列
sb.append(" "+fieldType+"("+field.getFieldLength()+","+ field.getDecimalPoint() +") ");
}else if ("decimal".equals(fieldType)){
// 追加列
sb.append(" "+fieldType+"("+field.getFieldLength()+","+ field.getDecimalPoint() +") ");
}else {
// 追加列
sb.append(" "+fieldType+"("+field.getFieldLength()+") ");
}
// 判断是否为主键 - 等于1是主键
if ("1".equals(field.getPrimaryKey())){
// 字段名称放进去
priKeyList.add(field.getFieldName());
// 判断是否允许为空 等于1是允许为空; 只有不为空的时候，需要设置
if (!"1".equals(field.getIsNull())){
sb.append("NOT NULL COMMENT '"+field.getFieldRemark()+"',\n");
}
// 如果到了最后一条，并且只有一个主键时
if (i >= tableFields.size()-1 && priKeyList.size() == 1){
sb.append("PRIMARY KEY (`"+ priKeyList.get(0) +"`)");
sb.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
}else if (i >= tableFields.size() -1 && priKeyList.size() > 1){
// 最后一条，并且存在多个主键时
sb.append("PRIMARY KEY (");
// 遍历主键集合
for (int j = 0; j < priKeyList.size(); j++) {
// 最后一个时
if (j == priKeyList.size() -1){
sb.append("`"+ priKeyList.get(j) +"`) USING BTREE \n");
}else {
sb.append("`"+ priKeyList.get(j) +"`,");
}
}
sb.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
}
// 非主键，直接判断是否允许为空
}else {
// 存在主键，并且为最后一个了
if (priKeyList.size() > 0 && i >= tableFields.size() -1 ){
// 判断是否为空 if是可以为空
if ("1".equals(field.getIsNull())){
sb.append("DEFAULT NULL COMMENT '"+ field.getFieldRemark() +"',\n");
}else {
sb.append("NOT NULL COMMENT '"+ field.getFieldRemark() +"',\n");
}
// 表示只有一个主键
if (priKeyList.size() == 1){
sb.append("PRIMARY KEY (`"+ priKeyList.get(0) +"`)\n");
sb.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
}else {
// 最后一条，并且存在多个主键时
sb.append("PRIMARY KEY (");
// 遍历主键集合
for (int j = 0; j < priKeyList.size(); j++) {
// 最后一个时
if (j == priKeyList.size() -1){
sb.append("`"+ priKeyList.get(j) +"`) USING BTREE \n");
}else {
sb.append("`"+ priKeyList.get(j) +"`,");
}
}
sb.append(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
}
}else {
// 没有就追加 判断是否为空
if ("1".equals(field.getIsNull())){
sb.append("DEFAULT NULL COMMENT '"+ field.getFieldRemark() +"',\n");
}else {
sb.append("NOT NULL COMMENT '"+ field.getFieldRemark() +"',\n");
}
}
}
}
return sb.toString();
}
/**
* 创建插入数据的SQL
* @param tableName 表名
* @param tableColumns 列名
* @param DataList 数据
* @return
*/
public void insertDataSQL(String tableName, List<String> tableColumns, List<List<String>> DataList){
String sql = "INSERT INTO "+tableName+" VALUES ";
if(DataList.size()>0){
for (int i=0; i<DataList.size(); i++){
sql+= "("+DataList+"),";
}
}
if(sql.lastIndexOf(",")>-1){
sql = sql.substring(0,sql.lastIndexOf(","));
jdbcTemplate.execute(sql);
}
}
/**jdbcTemplate
* 判断类型
* varchar
* int
* double
* datetime
* decimal
* text
*
* @param type
* @return
*/
private static String judgeDataType(String type){
switch (type){
case TYPE_STRING:
return "varchar";
case TYPE_INTEGER:
return "int";
case TYPE_DOUBLE:
return "double";
case TYPE_DATETIME:
return "datetime";
case TYPE_BIGDECIMAL:
return "decimal";
case TYPE_TEXT:
return "text";
default :
return "varchar";
}
};
//新增Excel数据源
private String upload2Maas(InputStream inputStream, String hostAndPort, String filename) throws IOException {
AmosRequestContext robotAuthentication = SpringContextHelper.getBean(AmosRequestContext.class);
......@@ -70,7 +422,7 @@ public class ClientHandler<path> implements Runnable {
// String product = robotAuthentication.getProduct();
// String appKey = robotAuthentication.getAppKey();
String token = "1121d8d2-915f-453e-8e20-1558566cefbb"; // 如果token、失效，就去浏览器复制最新的token、
String token = "17c26e9a-6b54-448e-bea4-72c01990ae7e"; // 如果token、失效，就去浏览器复制最新的token、
String product = "AMOS_STUDIO_WEB";
String appKey = "AMOS_STUDIO";
......@@ -183,7 +535,6 @@ public class ClientHandler<path> implements Runnable {
return null;
}
//根据机器人生成的校验参数将其放入请求头
private HttpHeaders getHeader(String token, String product, String appKey, String hostAndPort, Boolean isUpload) {
HttpHeaders header = new HttpHeaders();
......@@ -208,7 +559,6 @@ public class ClientHandler<path> implements Runnable {
while ((nextByte = is.read()) != -1) {
char c = (char) nextByte;
sb.append(c);
if (sb.toString().contains(delimiter)) {
return;
}
......@@ -235,7 +585,6 @@ public class ClientHandler<path> implements Runnable {
}
public static String filenameFilter(InputStream is) throws IOException {
StringBuilder sb = new StringBuilder();
BufferedInputStream bis = new BufferedInputStream(is);
findDelimiter(bis, "##STATITLE##");
......@@ -269,7 +618,6 @@ public class ClientHandler<path> implements Runnable {
return new FileInputStream(fileTmpPath);
}
//截取文件流
public static InputStream intercept(InputStream is) throws IOException {
......@@ -337,7 +685,6 @@ public class ClientHandler<path> implements Runnable {
}
private static void requestInfoToSocketServer() {
try {
Socket socket = new Socket("127.0.0.1", 7777);
......
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/config/ExcelTool.java 0 → 100644
View file @ cae4ba2e
package com.yeejoin.amos.kgd.config;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.web.multipart.MultipartFile;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.NumberFormat;
import java.util.*;
public class ExcelTool {
private static NumberFormat numberFormat = null;
/**
* 解析xls和xlsx不兼容问题
*
* @param pfs
* @param workbook
* @param inputStream
* @return
*/
public static Workbook getWorkBook(POIFSFileSystem pfs, Workbook workbook, InputStream inputStream, String filename) throws IOException {
if (filename.endsWith("xls")) {
pfs = new POIFSFileSystem(inputStream);
workbook = new HSSFWorkbook(pfs);
return workbook;
} else if (filename.endsWith("xlsx")) {
try {
workbook = new XSSFWorkbook(inputStream);
return workbook;
} catch (IOException e) {
return null;
}
} else {
return null;
}
}
/**
* 解析Excel表头
*
* @param inputStream 文件
* @return
*/
public static List<String> readColumnsFromExcel(InputStream inputStream, String filename) throws Exception {
POIFSFileSystem pfs = null;
Workbook workbook = null;
try {
// 解析xls和xlsx不兼容问题
workbook = getWorkBook(pfs, workbook, inputStream, filename);
} catch (IOException e) {
e.printStackTrace();
throw new Exception("模板保存异常。");
}
if (workbook == null) {
throw new Exception("请使用模板上传文件");
}
// 拿第一个sheet表
Sheet sheet = workbook.getSheetAt(0);
// 校验sheet是否合法
if (Objects.isNull(sheet)) {
return null;
}
// 获取第一行数据（假如第一行就是列名）
Row sheetTitleRow = sheet.getRow(sheet.getFirstRowNum());
// 取出最后一列
short lastCellNum = sheetTitleRow.getLastCellNum();
List<String> sheetTitleList = new LinkedList<>();
for (int i = 0; i < lastCellNum; i++) {
// 取出每一列的名
String cellValue = sheetTitleRow.getCell(i).getStringCellValue();
sheetTitleList.add(cellValue);
}
return sheetTitleList;
}
/**
* 解析Excel数据
*
* @param inputStream 文件
* @return
*/
public static List<List<String>> readDataFromExcel(InputStream inputStream, String filename) throws Exception {
POIFSFileSystem pfs = null;
Workbook workbook = null;
try {
// 解析xls和xlsx不兼容问题
workbook = getWorkBook(pfs, workbook, inputStream, filename);
} catch (IOException e) {
e.printStackTrace();
throw new Exception("模板保存异常。");
}
if (workbook == null) {
throw new Exception("请使用模板上传文件");
}
numberFormat = NumberFormat.getNumberInstance();
// 创建一个集合根据下标来确定每个单元格对应对象的什么属性
List<List<String>> resultList = new ArrayList<>();
// 获取表格第一个sheet的内容
Sheet sheetAt = workbook.getSheetAt(0);
// 获得sheet总行数
int lastRowNum = sheetAt.getLastRowNum();
if (lastRowNum < 1) {
throw new Exception("数据错误");
}
// 开始读取,不读取表头所以从第二行开始
for (int i = 1; i <= lastRowNum; i++) {
// 获取每一行
Row row = sheetAt.getRow(i);
// 行为空不读取
if (row == null) {
continue;
}
Cell cell = row.getCell(0);
//列为空不读取
if (cell == null || StringUtils.isEmpty(convertData(cell))) {
continue;
}
List<String> rowList = new ArrayList<>();
//添加数据
short lastCellNum = row.getLastCellNum();
for (int j = 0; j < lastCellNum; j++) {
Cell cellOne = row.getCell(j);
try {
String item = convertData(cellOne);
rowList.add(item.trim());
} catch (Exception e) {
System.out.println("-------------------Err-----------------------");
System.out.println(i + "行" + j + "列数据转换出现异常");
rowList.add("");
}
resultList.add(rowList);
}
//规避行数数据后几行为空
if (rowList.size() < lastCellNum) {
for (int k = 0; k < 15 - rowList.size(); k++) {
rowList.add("");
}
}
}
return resultList;
}
/**
* 表格数据转换
*
* @param cell
* @return
*/
public static String convertData(Cell cell) {
String str = "";
switch (cell.getCellTypeEnum()) {
case NUMERIC:
//判断是否是整数
str = numberFormat.format(cell.getNumericCellValue());
break;
case STRING:
str = cell.getStringCellValue();
break;
case FORMULA:
try {
str = String.valueOf(cell.getNumericCellValue());
} catch (IllegalArgumentException e) {
str = String.valueOf(cell.getRichStringCellValue());
}
break;
default:
str = "";
}
return str;
}
}
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/message/model/TableFieldModel.java 0 → 100644
View file @ cae4ba2e
package com.yeejoin.amos.kgd.message.model;
import lombok.Data;
@Data
public class TableFieldModel {
// 字段类型
private String fieldType="String";
// 字段名称
private String fieldName;
// 字段长度
private String fieldLength;
// 数值类型时，小数点长度
private String decimalPoint;
// 主键
private String primaryKey;
// 是否为空
private String isNull="1";
// 不为空时，需设置得值
private String fieldRemark;
}
amos-boot-module/amos-boot-module-api/amos-boot-module-kgd-api/src/main/java/com/yeejoin/amos/kgd/message/model/TableModel.java 0 → 100644
View file @ cae4ba2e
package com.yeejoin.amos.kgd.message.model;
import lombok.Data;
import java.util.List;
@Data
public class TableModel {
// 表名称
private String tableName;
// 表字段
private List<TableFieldModel> tableFields;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment