From 340852de2083d1090888deefd0bd3dd833f3bfd1 Mon Sep 17 00:00:00 2001 From: liulin <lin.liu@aliyun.com> Date: 星期一, 12 八月 2024 22:57:57 +0800 Subject: [PATCH] gauss导入导出 --- src/test/java/com/lunhan/xxx/host/GenCodeGauss.java | 157 ++++++++++++++++++------- src/test/java/com/lunhan/xxx/host/Mysql2Gauss.java | 69 ++++++----- src/test/java/com/lunhan/xxx/host/Gauss2Gauss.java | 111 ++++++++++++++++++ 3 files changed, 258 insertions(+), 79 deletions(-) diff --git a/src/test/java/com/lunhan/xxx/host/Gauss2Gauss.java b/src/test/java/com/lunhan/xxx/host/Gauss2Gauss.java new file mode 100644 index 0000000..3376194 --- /dev/null +++ b/src/test/java/com/lunhan/xxx/host/Gauss2Gauss.java @@ -0,0 +1,111 @@ +package com.lunhan.xxx.host; + +import com.lunhan.xxx.common.util.ListUtil; +import com.lunhan.xxx.common.util.StringUtil; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class Gauss2Gauss { + private static final String FROM_HOST = "113.250.189.120"; + private static final String FROM_PORT = "57654"; + private static final String FROM_USER = "lunhan"; + private static final String FROM_PASSWORD = "lunhan.20240330"; + private static final String FROM_DB_NAME = "water_trade_center"; + private static final String FROM_DRIVER = "org.postgresql.Driver"; + private static final String FROM_CONNECTION_STR = "jdbc:postgresql://" + FROM_HOST + ":" + FROM_PORT + "/" + FROM_DB_NAME; + + private static final String TO_HOST = "113.250.189.120"; + private static final String TO_PORT = "57654"; + private static final String TO_USER = "lunhan"; + private static final String TO_PASSWORD = "lunhan.20240330"; + private static final String TO_DB_NAME = "qinghe_wise_water_system"; + private static final String TO_DRIVER = "org.postgresql.Driver"; + private static final String TO_CONNECTION_STR = "jdbc:postgresql://" + TO_HOST + ":" + TO_PORT + "/" + TO_DB_NAME; + + private static Connection CONN_FROM = null; + private static Connection CONN_TO = null; + + static { + //加载驱动 + try { + Class.forName(FROM_DRIVER); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + //获得连接 + try { + CONN_FROM = DriverManager.getConnection(FROM_CONNECTION_STR, FROM_USER, FROM_PASSWORD); + } catch (SQLException e) { + e.printStackTrace(); + } + //加载驱动 + try { + Class.forName(TO_DRIVER); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + //获得连接 + try { + CONN_TO = DriverManager.getConnection(TO_CONNECTION_STR, TO_USER, TO_PASSWORD); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) throws SQLException { + // 创建触发器需要的函数 + GenCodeGauss.execSql(CONN_TO, GenCodeGauss.getAutoUpdateTimeFunction()); + + // 获取mysql所有的表名称 + Map<String, String> tables = GenCodeGauss.getTables(CONN_FROM, FROM_DB_NAME); + // 遍历表名,获取表结构 + for (Map.Entry<String, String> table : tables.entrySet()) { + // 获取mysql表结构 + List<Map<String, Object>> columns = GenCodeGauss.getColumns(CONN_FROM, table.getKey()); + // 获取mysql表的主键 + Map<String, String> mapPK = GenCodeGauss.getPK(CONN_FROM, table.getKey()); + // 获取mysql表的索引 + Map<String, String> mapIndex = GenCodeGauss.getIndex(CONN_FROM, table.getKey()); + + System.out.println("从mysql查询表结构:" + table.getKey()); + // 调用gauss创建表结构 + List<String> listGenTable = GenCodeGauss.genTable(table.getKey(), table.getValue(), columns, mapPK, mapIndex); +// for (String sql : listGenTable) { +// boolean result = GenCodeGauss.execSql(CONN_TO, sql); +// System.out.println(sql + ", 执行结果:" + result); +// } + + // 获取mysql表所有数据 + String sqlSelect = "SELECT * FROM " + table.getKey(); + Integer count = 1; + Integer pageSize = 500; + String orderBy = "id"; + Object lastId = null; + while (count > 0) { + count = 0; + String sql = sqlSelect + (Objects.isNull(lastId) ? "" : " WHERE " + orderBy + " > " + GenCodeGauss.toDBValue(lastId) + "") + " ORDER BY " + orderBy + " LIMIT " + pageSize; + List<Map<String, Object>> list = GenCodeGauss.getListAll(CONN_FROM, sql); + if (ListUtil.isNullOrEmpty(list)) { + continue; + } + count = list.size(); + lastId = list.get(list.size() - 1).get(orderBy); + + System.out.println("从mysql获取到数据:" + count); + // 批量插入gauss数据库 + List<String> insertAll = GenCodeGauss.insertAll(list, table.getKey(), 1); +// for (String addData : insertAll) { +// boolean result = GenCodeGauss.execSql(CONN_TO, addData); +// System.out.println(addData + ", 执行结果:" + result); +// } + String sqlInsert = StringUtil.join(insertAll, "\n"); + System.out.println(sqlInsert); + } + } + } +} diff --git a/src/test/java/com/lunhan/xxx/host/GenCodeGauss.java b/src/test/java/com/lunhan/xxx/host/GenCodeGauss.java index 42de15a..89ff2a8 100644 --- a/src/test/java/com/lunhan/xxx/host/GenCodeGauss.java +++ b/src/test/java/com/lunhan/xxx/host/GenCodeGauss.java @@ -1594,24 +1594,25 @@ return result; } - public static boolean genTable(Connection connect, String tableName, String tableDesc, List<Map<String, Object>> listColumn, Map<String, String> mapPK, Map<String, String> mapIndex) throws SQLException { + public static List<String> genTable(String tableName, String tableDesc, List<Map<String, Object>> listColumn, Map<String, String> mapPK, Map<String, String> mapIndex) { + List<String> list = new ArrayList<>(); + String sql = "CREATE TABLE \"public\".\"" + tableName + "\" (\n"; Map<String, String> mapColumnDesc = new HashMap<>(); int idx = 0; - boolean result = Boolean.FALSE; - result = GenCodeGauss.execSql(connect, "DROP TABLE IF EXISTS \"public\".\"" + tableName + "\";\n"); + list.add("-- \"" + tableName + "\" - \"" + tableDesc + "\"\nDROP TABLE IF EXISTS \"public\".\"" + tableName + "\";\n"); // 遍历列,构建sql for (Map<String, Object> column : listColumn) { idx++; String name = column.get("name").toString(); String type = column.get("type").toString(); - String desc = column.get("description").toString(); - String isNullable = column.get("isNullable").toString(); + Object desc = column.get("description"); + Object isNullable = column.get("isNullable"); - mapColumnDesc.put(name, desc); + mapColumnDesc.put(name, ParameterUtil.dealNullStr(desc)); sql += "\t\"" + name + "\" " + GenCodeGauss.mysqlType2Gauss(type); switch (type) { @@ -1672,7 +1673,7 @@ } sql += ")\n" + ";\n"; - result = GenCodeGauss.execSql(connect, sql); + list.add(sql); sql = ""; sql += "COMMENT ON TABLE \"public\".\"" + tableName + "\" IS '" + tableDesc + "';\n"; @@ -1680,7 +1681,7 @@ for (Map.Entry<String, String> desc : mapColumnDesc.entrySet()) { sql += "COMMENT ON COLUMN \"public\".\"" + tableName + "\".\"" + desc.getKey() + "\" IS '" + desc.getValue().replace("\r\n", " ").replace("\n", " ") + "';\n"; } - result = GenCodeGauss.execSql(connect, sql); + list.add("-- 字段描述\n" + sql); sql = ""; // 设置主键 @@ -1690,25 +1691,25 @@ } } sql += "ALTER TABLE \"public\".\"" + tableName + "\" ADD CONSTRAINT \"PK_" + tableName + "\" PRIMARY KEY (\"id\");\n"; - result = GenCodeGauss.execSql(connect, sql); + list.add("-- 设置主键\n" + sql); sql = ""; // 触发器:自动赋值最后更新时间 sql += "CREATE TRIGGER \"TG_" + tableName + "\" BEFORE INSERT OR UPDATE ON \"public\".\"" + tableName + "\"\n" + "FOR EACH ROW\n" + "EXECUTE PROCEDURE \"public\".\"update_timestamp_column\"();"; - result = GenCodeGauss.execSql(connect, sql); + list.add("-- 触发器:自动赋值最后更新时间\n" + sql); sql = ""; - + list.add("-- 创建索引\n"); // 遍历索引,构建sql for (Map.Entry<String, String> index : mapIndex.entrySet()) { if (!mapPK.containsKey(index.getKey())) { sql += "CREATE INDEX \"" + index.getKey() + "\" ON \"public\".\"test_info\"(\"" + index.getValue() + "\");\n"; - result = GenCodeGauss.execSql(connect, sql); + list.add(sql); sql = ""; } } - return result; + return list; } private static String mysqlType2Gauss(String type) { @@ -1802,9 +1803,8 @@ return mapPK; } - public static List<Map<String, Object>> getListAll(Connection connect, String tableName, String dbSchema) { + public static List<Map<String, Object>> getListAll(Connection connect, String sql) { List<Map<String, Object>> result = new ArrayList<>(); - String sql = "SELECT * FROM " + dbSchema + "." + tableName; try (PreparedStatement pStmt = connect.prepareStatement(sql)) { try (ResultSet rs = pStmt.executeQuery()) { //获取键名 @@ -1827,8 +1827,8 @@ return result; } - public static List<Boolean> insertAll(Connection connect, List<Map<String, Object>> list, String tableName, Integer onceCount) { - List<Boolean> result = new ArrayList<>(); + public static List<String> insertAll(List<Map<String, Object>> list, String tableName, Integer onceCount) { + List<String> result = new ArrayList<>(); String insertSql = "INSERT INTO public." + tableName + "(\""; List<String> listColumn = new ArrayList<>(); for (String column : list.get(0).keySet()) { @@ -1840,7 +1840,6 @@ insertSql += StringUtil.join(listColumn, "\",\"") + "\") VALUES"; String sql = ""; - List<Object> listValue = new ArrayList<>(); for (int i = 0; i < list.size(); i++) { if (StringUtil.isNullOrEmpty(sql)) { sql = insertSql; @@ -1854,43 +1853,107 @@ } listColValue.add(item.getValue()); } - sql += "\n(" + StringUtil.join(listColValue.stream().map(c -> "?").collect(Collectors.toList())) + "),"; - listValue.addAll(listColValue); + sql += "\n(" + StringUtil.join(listColValue.stream().map(GenCodeGauss::getSqlValue).collect(Collectors.toList())) + "),"; + if (i > 0 && i % onceCount == 0) { sql = sql.substring(0, sql.length() - 1); - try (PreparedStatement pStmt = connect.prepareStatement(sql)) { - for (int j = 0; j < listValue.size(); j++) { - pStmt.setObject(j + 1, listValue.get(j)); - } - listValue = new ArrayList<>(); - boolean insertList = pStmt.execute(); - result.add(insertList); - System.out.println("插入数据" + Math.max(i - onceCount, 0) + " - " + i + ",结果:" + insertList); - } catch (SQLException e) { - e.printStackTrace(); - } + + result.add(sql + ";"); + sql = ""; } } if (StringUtil.isNotNullOrEmpty(sql)) { sql = sql.substring(0, sql.length() - 1); - try (PreparedStatement pStmt = connect.prepareStatement(sql)) { - for (int j = 0; j < listValue.size(); j++) { - Object value = listValue.get(j); - if (Objects.equals("", value)) { - // TODO 神奇的gauss,''会判定为null,如果有非空约束,会插入失败 - value = " "; - } - pStmt.setObject(j + 1, value); - } - listValue = new ArrayList<>(); - boolean insertList = pStmt.execute(); - result.add(insertList); - System.out.println("插入数据" + (list.size() - list.size() % onceCount) + " - " + list.size() + ",结果:" + insertList); - } catch (SQLException e) { - e.printStackTrace(); - } + + result.add(sql + ";"); } return result; } + private static String getSqlValue(Object value) { + if (Objects.isNull(value)) { + return "null"; + } + if (value instanceof String) { + return "'" + value + "'"; + } + if (value instanceof Date) { + return "'" + LocalDateTimeUtil.toFormatFullString(((Date)value).getTime()) + "'"; + } + if (value instanceof LocalDate) { + return "'" + LocalDateTimeUtil.toFormatString((LocalDate)value) + "'"; + } + if (value instanceof LocalDateTime) { + return "'" + LocalDateTimeUtil.toFormatFullString((LocalDateTime)value) + "'"; + } + if (value instanceof Calendar) { + return "'" + CalendarUtil.toDateTimeMSStr((Calendar)value) + "'"; + } + return value.toString(); + } + + public static String getAutoUpdateTimeFunction() { + return "CREATE OR REPLACE FUNCTION \"public\".\"update_timestamp_column\"()\n" + + " RETURNS \"pg_catalog\".\"trigger\" AS $BODY$\n" + + "BEGIN\n" + + "NEW.update_time := current_timestamp;\n" + + "RETURN NEW;\n" + + "END;\n" + + "$BODY$\n" + + " LANGUAGE plpgsql VOLATILE\n" + + " COST 100"; + } + + public static List<String> export(Connection connect, String dbSchema, boolean isData, boolean isAutoUpdateTimeFunc) { + List<String> listTable = new ArrayList<>(); + List<String> listData = new ArrayList<>(); + if (isAutoUpdateTimeFunc) { + listTable.add(GenCodeGauss.getAutoUpdateTimeFunction()); + } + // 获取所有表 + Map<String, String> tables = GenCodeGauss.getTables(connect, dbSchema); + // 遍历所有表 + for (Map.Entry<String, String> table : tables.entrySet()) { + String tabName = table.getKey(); + String tabDesc = table.getValue(); + + // 获取所有列 + List<Map<String, Object>> listCol = GenCodeGauss.getColumns(connect, tabName); + + Map<String, String> mapPK = GenCodeGauss.getPK(connect, tabName); + + Map<String, String> mapIndex = GenCodeGauss.getIndex(connect, tabName); + + // 生成表的sql + List<String> listGenTable = GenCodeGauss.genTable(tabName, tabDesc, listCol, mapPK, mapIndex); + listTable.addAll(listGenTable); + + if (isData) { + // 获取mysql表所有数据 + String sqlSelect = "SELECT * FROM " + table.getKey(); + Integer count = 1; + Integer pageSize = ConstantFactory.NUM500; + String orderBy = "id"; + Object lastId = null; + while (count > 0) { + count = 0; + String sql = sqlSelect + (Objects.isNull(lastId) ? "" : " WHERE " + orderBy + " > " + GenCodeGauss.toDBValue(lastId) + "") + " ORDER BY " + orderBy + " LIMIT " + pageSize; + List<Map<String, Object>> list = GenCodeGauss.getListAll(connect, sql); + if (ListUtil.isNullOrEmpty(list)) { + continue; + } + count = list.size(); + lastId = list.get(list.size() - 1).get(orderBy); + + // 批量插入gauss数据库 + List<String> insertAll = GenCodeGauss.insertAll(list, table.getKey(), ConstantFactory.NUM50); + listData.add(StringUtil.join(insertAll, "\n")); + } + } + } + if (ListUtil.isNotNullOrEmpty(listData)) { + listTable.addAll(listData); + } + return listTable; + } } diff --git a/src/test/java/com/lunhan/xxx/host/Mysql2Gauss.java b/src/test/java/com/lunhan/xxx/host/Mysql2Gauss.java index dbd0ac2..24ea4cf 100644 --- a/src/test/java/com/lunhan/xxx/host/Mysql2Gauss.java +++ b/src/test/java/com/lunhan/xxx/host/Mysql2Gauss.java @@ -1,7 +1,6 @@ package com.lunhan.xxx.host; import com.lunhan.xxx.common.util.ListUtil; -import com.lunhan.xxx.common.util.SerializeUtil; import java.sql.Connection; import java.sql.DriverManager; @@ -11,47 +10,47 @@ import java.util.Objects; public class Mysql2Gauss { - private static final String Mysql_HOST = "113.250.189.120"; - private static final String Mysql_PORT = "23306"; - private static final String Mysql_USER = "lunhan"; - private static final String Mysql_PASSWORD = "lunhan123"; - private static final String Mysql_DB_NAME = "water_basic"; - private static final String Mysql_DRIVER = "com.mysql.cj.jdbc.Driver"; - private static final String Mysql_CONNECTION_STR = "jdbc:mysql://" + Mysql_HOST + ":" + Mysql_PORT + "/" + Mysql_DB_NAME + "?serverTimezone=UTC&useUnicode=true&characterEncoding=utf8&useSSL=false"; + private static final String FROM_HOST = "113.250.189.120"; + private static final String FROM_PORT = "23306"; + private static final String FROM_USER = "lunhan"; + private static final String FROM_PASSWORD = "lunhan123"; + private static final String FROM_DB_NAME = "water_basic"; + private static final String FROM_DRIVER = "com.mysql.cj.jdbc.Driver"; + private static final String FROM_CONNECTION_STR = "jdbc:mysql://" + FROM_HOST + ":" + FROM_PORT + "/" + FROM_DB_NAME + "?serverTimezone=UTC&useUnicode=true&characterEncoding=utf8&useSSL=false"; - private static final String Gauss_HOST = "113.250.189.120"; - private static final String Gauss_PORT = "57654"; - private static final String Gauss_USER = "lunhan"; - private static final String Gauss_PASSWORD = "lunhan.20240330"; - private static final String Gauss_DB_NAME = "water_basic"; - private static final String Gauss_DRIVER = "org.postgresql.Driver"; - private static final String Gauss_CONNECTION_STR = "jdbc:postgresql://" + Gauss_HOST + ":" + Gauss_PORT + "/" + Gauss_DB_NAME; + private static final String TO_HOST = "113.250.189.120"; + private static final String TO_PORT = "57654"; + private static final String TO_USER = "lunhan"; + private static final String TO_PASSWORD = "lunhan.20240330"; + private static final String TO_DB_NAME = "water_basic"; + private static final String TO_DRIVER = "org.postgresql.Driver"; + private static final String TO_CONNECTION_STR = "jdbc:postgresql://" + TO_HOST + ":" + TO_PORT + "/" + TO_DB_NAME; - private static Connection CONN_Mysql = null; - private static Connection CONN_Gauss = null; + private static Connection CONN_FROM = null; + private static Connection CONN_TO = null; static { //加载驱动 try { - Class.forName(Mysql_DRIVER); + Class.forName(FROM_DRIVER); } catch (ClassNotFoundException e) { e.printStackTrace(); } //获得连接 try { - CONN_Mysql = DriverManager.getConnection(Mysql_CONNECTION_STR, Mysql_USER, Mysql_PASSWORD); + CONN_FROM = DriverManager.getConnection(FROM_CONNECTION_STR, FROM_USER, FROM_PASSWORD); } catch (SQLException e) { e.printStackTrace(); } //加载驱动 try { - Class.forName(Gauss_DRIVER); + Class.forName(TO_DRIVER); } catch (ClassNotFoundException e) { e.printStackTrace(); } //获得连接 try { - CONN_Gauss = DriverManager.getConnection(Gauss_CONNECTION_STR, Gauss_USER, Gauss_PASSWORD); + CONN_TO = DriverManager.getConnection(TO_CONNECTION_STR, TO_USER, TO_PASSWORD); } catch (SQLException e) { e.printStackTrace(); } @@ -59,7 +58,7 @@ public static void main(String[] args) throws SQLException { // 创建触发器需要的函数 - GenCodeGauss.execSql(CONN_Gauss, "CREATE OR REPLACE FUNCTION \"public\".\"update_timestamp_column\"()\n" + + GenCodeGauss.execSql(CONN_TO, "CREATE OR REPLACE FUNCTION \"public\".\"update_timestamp_column\"()\n" + " RETURNS \"pg_catalog\".\"trigger\" AS $BODY$\n" + "BEGIN\n" + "NEW.update_time := current_timestamp;\n" + @@ -70,23 +69,26 @@ " COST 100"); // 获取mysql所有的表名称 - Map<String, String> tables = GenCodeMysql.getTables(CONN_Mysql, Mysql_DB_NAME); + Map<String, String> tables = GenCodeMysql.getTables(CONN_FROM, FROM_DB_NAME); // 遍历表名,获取表结构 for (Map.Entry<String, String> table : tables.entrySet()) { // 获取mysql表结构 - List<Map<String, Object>> columns = GenCodeMysql.getColumns(CONN_Mysql, table.getKey(), Mysql_DB_NAME); + List<Map<String, Object>> columns = GenCodeMysql.getColumns(CONN_FROM, table.getKey(), FROM_DB_NAME); // 获取mysql表的主键 - Map<String, String> mapPK = GenCodeMysql.getPK(CONN_Mysql, table.getKey()); + Map<String, String> mapPK = GenCodeMysql.getPK(CONN_FROM, table.getKey()); // 获取mysql表的索引 - Map<String, String> mapIndex = GenCodeMysql.getIndex(CONN_Mysql, table.getKey()); + Map<String, String> mapIndex = GenCodeMysql.getIndex(CONN_FROM, table.getKey()); System.out.println("从mysql查询表结构:" + table.getKey()); // 调用gauss创建表结构 - boolean genTale = GenCodeGauss.genTable(CONN_Gauss, table.getKey(), table.getValue(), columns, mapPK, mapIndex); - System.out.println("向gauss数据库创建表结果:" + genTale); + List<String> listGenTable = GenCodeGauss.genTable(table.getKey(), table.getValue(), columns, mapPK, mapIndex); + for (String sql : listGenTable) { + boolean result = GenCodeGauss.execSql(CONN_TO, sql); + System.out.println(sql + ", 执行结果:" + result); + } // 获取mysql表所有数据 - String sqlSelect = "SELECT * FROM " + Mysql_DB_NAME + "." + table.getKey(); + String sqlSelect = "SELECT * FROM " + FROM_DB_NAME + "." + table.getKey(); Integer count = 1; Integer pageSize = 500; String orderBy = "id"; @@ -94,7 +96,7 @@ while (count > 0) { count = 0; String sql = sqlSelect + (Objects.isNull(lastId) ? "" : " WHERE " + orderBy + " > " + GenCodeGauss.toDBValue(lastId) + "") + " ORDER BY " + orderBy + " LIMIT " + pageSize; - List<Map<String, Object>> list = GenCodeMysql.getListAll(CONN_Mysql, sql); + List<Map<String, Object>> list = GenCodeMysql.getListAll(CONN_FROM, sql); if (ListUtil.isNullOrEmpty(list)) { continue; } @@ -103,8 +105,11 @@ System.out.println("从mysql获取到数据:" + count); // 批量插入gauss数据库 - List<Boolean> insertAll = GenCodeGauss.insertAll(CONN_Gauss, list, table.getKey(), pageSize); - System.out.println("向gauss数据库插入数据结果:" + SerializeUtil.toJson(insertAll)); + List<String> insertAll = GenCodeGauss.insertAll(list, table.getKey(), pageSize); + for (String addData : insertAll) { + boolean result = GenCodeGauss.execSql(CONN_TO, addData); + System.out.println(addData + ", 执行结果:" + result); + } } } } -- Gitblit v1.9.3