From 3d0a36b4f348c33deed83023b2da964f1b43b4b1 Mon Sep 17 00:00:00 2001 From: imhao183 Date: Sun, 24 Jan 2021 20:27:22 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=87=AA=E5=8A=A8=E6=B8=85?= =?UTF-8?q?=E7=90=86=E5=8E=86=E5=8F=B2=E6=B6=88=E6=81=AF=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../h2/V37__add_user_messages_mid_index.sql | 1 + .../V42__add_user_messages_mid_index.sql | 129 ++++++++++++++++ .../src/main/java/win/liyufan/im/DBUtil.java | 141 ++++-------------- 3 files changed, 160 insertions(+), 111 deletions(-) create mode 100755 broker/migrate/h2/V37__add_user_messages_mid_index.sql create mode 100755 broker/migrate/mysql/V42__add_user_messages_mid_index.sql diff --git a/broker/migrate/h2/V37__add_user_messages_mid_index.sql b/broker/migrate/h2/V37__add_user_messages_mid_index.sql new file mode 100755 index 00000000..0d440c40 --- /dev/null +++ b/broker/migrate/h2/V37__add_user_messages_mid_index.sql @@ -0,0 +1 @@ +alter table `t_user_messages` ADD INDEX `user_messages_mid_index` ( `_mid`); diff --git a/broker/migrate/mysql/V42__add_user_messages_mid_index.sql b/broker/migrate/mysql/V42__add_user_messages_mid_index.sql new file mode 100755 index 00000000..a6ab6f7e --- /dev/null +++ b/broker/migrate/mysql/V42__add_user_messages_mid_index.sql @@ -0,0 +1,129 @@ +alter table `t_user_messages` add INDEX `user_messages_mid_index` ( `_mid` ); +alter table `t_user_messages_0` add INDEX `user_messages0_mid_index` ( `_mid` ); +alter table `t_user_messages_1` add INDEX `user_messages1_mid_index` ( `_mid` ); +alter table `t_user_messages_2` add INDEX `user_messages2_mid_index` ( `_mid` ); +alter table `t_user_messages_3` add INDEX `user_messages3_mid_index` ( `_mid` ); +alter table `t_user_messages_4` add INDEX `user_messages4_mid_index` ( `_mid` ); +alter table `t_user_messages_5` add INDEX `user_messages5_mid_index` ( `_mid` ); +alter table `t_user_messages_6` add INDEX `user_messages6_mid_index` ( `_mid` ); +alter table `t_user_messages_7` add INDEX `user_messages7_mid_index` ( `_mid` ); +alter table `t_user_messages_8` add INDEX `user_messages8_mid_index` ( `_mid` ); +alter table `t_user_messages_9` add INDEX `user_messages9_mid_index` ( `_mid` ); +alter table `t_user_messages_10` add INDEX `user_messages10_mid_index` ( `_mid` ); +alter table `t_user_messages_11` add INDEX `user_messages11_mid_index` ( `_mid` ); +alter table `t_user_messages_12` add INDEX `user_messages12_mid_index` ( `_mid` ); +alter table `t_user_messages_13` add INDEX `user_messages13_mid_index` ( `_mid` ); +alter table `t_user_messages_14` add INDEX `user_messages14_mid_index` ( `_mid` ); +alter table `t_user_messages_15` add INDEX `user_messages15_mid_index` ( `_mid` ); +alter table `t_user_messages_16` add INDEX `user_messages16_mid_index` ( `_mid` ); +alter table `t_user_messages_17` add INDEX `user_messages17_mid_index` ( `_mid` ); +alter table `t_user_messages_18` add INDEX `user_messages18_mid_index` ( `_mid` ); +alter table `t_user_messages_19` add INDEX `user_messages19_mid_index` ( `_mid` ); +alter table `t_user_messages_20` add INDEX `user_messages20_mid_index` ( `_mid` ); +alter table `t_user_messages_21` add INDEX `user_messages21_mid_index` ( `_mid` ); +alter table `t_user_messages_22` add INDEX `user_messages22_mid_index` ( `_mid` ); +alter table `t_user_messages_23` add INDEX `user_messages23_mid_index` ( `_mid` ); +alter table `t_user_messages_24` add INDEX `user_messages24_mid_index` ( `_mid` ); +alter table `t_user_messages_25` add INDEX `user_messages25_mid_index` ( `_mid` ); +alter table `t_user_messages_26` add INDEX `user_messages26_mid_index` ( `_mid` ); +alter table `t_user_messages_27` add INDEX `user_messages27_mid_index` ( `_mid` ); +alter table `t_user_messages_28` add INDEX `user_messages28_mid_index` ( `_mid` ); +alter table `t_user_messages_29` add INDEX `user_messages29_mid_index` ( `_mid` ); +alter table `t_user_messages_30` add INDEX `user_messages30_mid_index` ( `_mid` ); +alter table `t_user_messages_31` add INDEX `user_messages31_mid_index` ( `_mid` ); +alter table `t_user_messages_32` add INDEX `user_messages32_mid_index` ( `_mid` ); +alter table `t_user_messages_33` add INDEX `user_messages33_mid_index` ( `_mid` ); +alter table `t_user_messages_34` add INDEX `user_messages34_mid_index` ( `_mid` ); +alter table `t_user_messages_35` add INDEX `user_messages35_mid_index` ( `_mid` ); +alter table `t_user_messages_36` add INDEX `user_messages36_mid_index` ( `_mid` ); +alter table `t_user_messages_37` add INDEX `user_messages37_mid_index` ( `_mid` ); +alter table `t_user_messages_38` add INDEX `user_messages38_mid_index` ( `_mid` ); +alter table `t_user_messages_39` add INDEX `user_messages39_mid_index` ( `_mid` ); +alter table `t_user_messages_40` add INDEX `user_messages40_mid_index` ( `_mid` ); +alter table `t_user_messages_41` add INDEX `user_messages41_mid_index` ( `_mid` ); +alter table `t_user_messages_42` add INDEX `user_messages42_mid_index` ( `_mid` ); +alter table `t_user_messages_43` add INDEX `user_messages43_mid_index` ( `_mid` ); +alter table `t_user_messages_44` add INDEX `user_messages44_mid_index` ( `_mid` ); +alter table `t_user_messages_45` add INDEX `user_messages45_mid_index` ( `_mid` ); +alter table `t_user_messages_46` add INDEX `user_messages46_mid_index` ( `_mid` ); +alter table `t_user_messages_47` add INDEX `user_messages47_mid_index` ( `_mid` ); +alter table `t_user_messages_48` add INDEX `user_messages48_mid_index` ( `_mid` ); +alter table `t_user_messages_49` add INDEX `user_messages49_mid_index` ( `_mid` ); +alter table `t_user_messages_50` add INDEX `user_messages50_mid_index` ( `_mid` ); +alter table `t_user_messages_51` add INDEX `user_messages51_mid_index` ( `_mid` ); +alter table `t_user_messages_52` add INDEX `user_messages52_mid_index` ( `_mid` ); +alter table `t_user_messages_53` add INDEX `user_messages53_mid_index` ( `_mid` ); +alter table `t_user_messages_54` add INDEX `user_messages54_mid_index` ( `_mid` ); +alter table `t_user_messages_55` add INDEX `user_messages55_mid_index` ( `_mid` ); +alter table `t_user_messages_56` add INDEX `user_messages56_mid_index` ( `_mid` ); +alter table `t_user_messages_57` add INDEX `user_messages57_mid_index` ( `_mid` ); +alter table `t_user_messages_58` add INDEX `user_messages58_mid_index` ( `_mid` ); +alter table `t_user_messages_59` add INDEX `user_messages59_mid_index` ( `_mid` ); +alter table `t_user_messages_60` add INDEX `user_messages60_mid_index` ( `_mid` ); +alter table `t_user_messages_61` add INDEX `user_messages61_mid_index` ( `_mid` ); +alter table `t_user_messages_62` add INDEX `user_messages62_mid_index` ( `_mid` ); +alter table `t_user_messages_63` add INDEX `user_messages63_mid_index` ( `_mid` ); +alter table `t_user_messages_64` add INDEX `user_messages64_mid_index` ( `_mid` ); +alter table `t_user_messages_65` add INDEX `user_messages65_mid_index` ( `_mid` ); +alter table `t_user_messages_66` add INDEX `user_messages66_mid_index` ( `_mid` ); +alter table `t_user_messages_67` add INDEX `user_messages67_mid_index` ( `_mid` ); +alter table `t_user_messages_68` add INDEX `user_messages68_mid_index` ( `_mid` ); +alter table `t_user_messages_69` add INDEX `user_messages69_mid_index` ( `_mid` ); +alter table `t_user_messages_70` add INDEX `user_messages70_mid_index` ( `_mid` ); +alter table `t_user_messages_71` add INDEX `user_messages71_mid_index` ( `_mid` ); +alter table `t_user_messages_72` add INDEX `user_messages72_mid_index` ( `_mid` ); +alter table `t_user_messages_73` add INDEX `user_messages73_mid_index` ( `_mid` ); +alter table `t_user_messages_74` add INDEX `user_messages74_mid_index` ( `_mid` ); +alter table `t_user_messages_75` add INDEX `user_messages75_mid_index` ( `_mid` ); +alter table `t_user_messages_76` add INDEX `user_messages76_mid_index` ( `_mid` ); +alter table `t_user_messages_77` add INDEX `user_messages77_mid_index` ( `_mid` ); +alter table `t_user_messages_78` add INDEX `user_messages78_mid_index` ( `_mid` ); +alter table `t_user_messages_79` add INDEX `user_messages79_mid_index` ( `_mid` ); +alter table `t_user_messages_80` add INDEX `user_messages80_mid_index` ( `_mid` ); +alter table `t_user_messages_81` add INDEX `user_messages81_mid_index` ( `_mid` ); +alter table `t_user_messages_82` add INDEX `user_messages82_mid_index` ( `_mid` ); +alter table `t_user_messages_83` add INDEX `user_messages83_mid_index` ( `_mid` ); +alter table `t_user_messages_84` add INDEX `user_messages84_mid_index` ( `_mid` ); +alter table `t_user_messages_85` add INDEX `user_messages85_mid_index` ( `_mid` ); +alter table `t_user_messages_86` add INDEX `user_messages86_mid_index` ( `_mid` ); +alter table `t_user_messages_87` add INDEX `user_messages87_mid_index` ( `_mid` ); +alter table `t_user_messages_88` add INDEX `user_messages88_mid_index` ( `_mid` ); +alter table `t_user_messages_89` add INDEX `user_messages89_mid_index` ( `_mid` ); +alter table `t_user_messages_90` add INDEX `user_messages90_mid_index` ( `_mid` ); +alter table `t_user_messages_91` add INDEX `user_messages91_mid_index` ( `_mid` ); +alter table `t_user_messages_92` add INDEX `user_messages92_mid_index` ( `_mid` ); +alter table `t_user_messages_93` add INDEX `user_messages93_mid_index` ( `_mid` ); +alter table `t_user_messages_94` add INDEX `user_messages94_mid_index` ( `_mid` ); +alter table `t_user_messages_95` add INDEX `user_messages95_mid_index` ( `_mid` ); +alter table `t_user_messages_96` add INDEX `user_messages96_mid_index` ( `_mid` ); +alter table `t_user_messages_97` add INDEX `user_messages97_mid_index` ( `_mid` ); +alter table `t_user_messages_98` add INDEX `user_messages98_mid_index` ( `_mid` ); +alter table `t_user_messages_99` add INDEX `user_messages99_mid_index` ( `_mid` ); +alter table `t_user_messages_100` add INDEX `user_messages100_mid_index` ( `_mid` ); +alter table `t_user_messages_101` add INDEX `user_messages101_mid_index` ( `_mid` ); +alter table `t_user_messages_102` add INDEX `user_messages102_mid_index` ( `_mid` ); +alter table `t_user_messages_103` add INDEX `user_messages103_mid_index` ( `_mid` ); +alter table `t_user_messages_104` add INDEX `user_messages104_mid_index` ( `_mid` ); +alter table `t_user_messages_105` add INDEX `user_messages105_mid_index` ( `_mid` ); +alter table `t_user_messages_106` add INDEX `user_messages106_mid_index` ( `_mid` ); +alter table `t_user_messages_107` add INDEX `user_messages107_mid_index` ( `_mid` ); +alter table `t_user_messages_108` add INDEX `user_messages108_mid_index` ( `_mid` ); +alter table `t_user_messages_109` add INDEX `user_messages109_mid_index` ( `_mid` ); +alter table `t_user_messages_110` add INDEX `user_messages110_mid_index` ( `_mid` ); +alter table `t_user_messages_111` add INDEX `user_messages111_mid_index` ( `_mid` ); +alter table `t_user_messages_112` add INDEX `user_messages112_mid_index` ( `_mid` ); +alter table `t_user_messages_113` add INDEX `user_messages113_mid_index` ( `_mid` ); +alter table `t_user_messages_114` add INDEX `user_messages114_mid_index` ( `_mid` ); +alter table `t_user_messages_115` add INDEX `user_messages115_mid_index` ( `_mid` ); +alter table `t_user_messages_116` add INDEX `user_messages116_mid_index` ( `_mid` ); +alter table `t_user_messages_117` add INDEX `user_messages117_mid_index` ( `_mid` ); +alter table `t_user_messages_118` add INDEX `user_messages118_mid_index` ( `_mid` ); +alter table `t_user_messages_119` add INDEX `user_messages119_mid_index` ( `_mid` ); +alter table `t_user_messages_120` add INDEX `user_messages120_mid_index` ( `_mid` ); +alter table `t_user_messages_121` add INDEX `user_messages121_mid_index` ( `_mid` ); +alter table `t_user_messages_122` add INDEX `user_messages122_mid_index` ( `_mid` ); +alter table `t_user_messages_123` add INDEX `user_messages123_mid_index` ( `_mid` ); +alter table `t_user_messages_124` add INDEX `user_messages124_mid_index` ( `_mid` ); +alter table `t_user_messages_125` add INDEX `user_messages125_mid_index` ( `_mid` ); +alter table `t_user_messages_126` add INDEX `user_messages126_mid_index` ( `_mid` ); +alter table `t_user_messages_127` add INDEX `user_messages127_mid_index` ( `_mid` ); diff --git a/broker/src/main/java/win/liyufan/im/DBUtil.java b/broker/src/main/java/win/liyufan/im/DBUtil.java index 563cfae0..844cc3de 100755 --- a/broker/src/main/java/win/liyufan/im/DBUtil.java +++ b/broker/src/main/java/win/liyufan/im/DBUtil.java @@ -166,133 +166,52 @@ public class DBUtil { return SystemExiting; } - private static void clearH2DB() { - boolean hasMore = true; - while (hasMore) { - hasMore = clearOneTable("t_user_messages"); - if(sleep()) { - return; - } - } - hasMore = true; - - while (hasMore) { - hasMore = clearOneTable("t_messages"); - if(sleep()) { - return; - } - } - } - private static void clearMySQL() { - int i = 0; - do { - if(!clearOneTable("t_user_messages_" + i)) { - i++; - } - - if(sleep()) { - return; - } - } while (i < 128); - - boolean hasMore = true; - String msgTableName; - if(ClearDBDebugMode) { - msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis())); - } else { - msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis()), 1); - } - while (hasMore) { - hasMore = clearOneTable(msgTableName); - if(sleep()) { - return; - } - } - + private static long getMaxMidNeedClean() { Calendar cal = Calendar.getInstance(); cal.setTime(new Date()); - if(cal.get(Calendar.DAY_OF_MONTH) > 25 && !ClearDBDebugMode) { - hasMore = true; - msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis()), 2); - while (hasMore) { - hasMore = clearOneTable(msgTableName); - if(sleep()) { - return; - } - } - } + cal.add(Calendar.YEAR, -3); + cal.add(Calendar.DATE, 1); + return MessageShardingUtil.getMsgIdFromTimestamp(cal.getTimeInMillis()); } - private static boolean clearOneTable(String tableName) { + private static void clearH2DB() { + clearOneTable("t_user_messages"); + if(sleep()) { + return; + } + clearOneTable("t_messages"); + } + + private static void clearMySQL() { + for (int i = 0; i < 128; i++) { + clearOneTable("t_user_messages_" + i); + if(sleep()) { + return; + } + } + + String msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis())); + clearOneTable(msgTableName); + } + + private static void clearOneTable(String tableName) { Connection connection = null; PreparedStatement statement = null; ResultSet rs = null; - int clearCount = 999; - if(ClearDBDebugMode) { - clearCount = 19; - } - boolean hasMore = true; + long mid = getMaxMidNeedClean(); + try { connection = DBUtil.getConnection(); - String sql = "select id, _mid from " + tableName + " order by id limit 1 offset " + clearCount; + String sql = "delete from " + tableName + " where _mid <= " + mid; statement = connection.prepareStatement(sql); - - rs = statement.executeQuery(); - int id = 0; - long mid = 0; - if (rs.next()) { - id = rs.getInt(1); - mid = rs.getLong(2); - } - if(id == 0) { - hasMore = false; - rs.close(); - rs = null; - statement.close(); - - sql = "select id, _mid from " + tableName + " order by id desc limit 1"; - statement = connection.prepareStatement(sql); - - rs = statement.executeQuery(); - if (rs.next()) { - id = rs.getInt(1); - mid = rs.getLong(2); - } - } - - rs.close(); - rs = null; - LOG.info("Check history message in table {}, id {}, mid {}, hasMore {}", tableName, id, mid, hasMore); - if(id != 0) { - Calendar tm = MessageShardingUtil.getCalendarFromMessageId(mid); - if(ClearDBDebugMode) { - tm.add(Calendar.HOUR, 1); - } else { - tm.add(Calendar.YEAR, 3); - tm.add(Calendar.MONTH, -1); - tm.add(Calendar.DATE, -4); - } - if(tm.getTimeInMillis() < System.currentTimeMillis()) { - statement.close(); - sql = "delete from " + tableName + " where id <= " + id; - statement = connection.prepareStatement(sql); - int count = statement.executeUpdate(); - LOG.info("Clear {} rows", count); - } else { - LOG.info("No history messages need clean"); - hasMore = false; - } - } else { - LOG.info("No messages"); - } + int count = statement.executeUpdate(); + LOG.info("Delete history message {} rows", count); } catch (Exception e) { - hasMore = false; e.printStackTrace(); Utility.printExecption(LOG, e, RDBS_Exception); } finally { DBUtil.closeDB(connection, statement, rs); } - return hasMore; } private static List getCreateSql() {