anna/IM-server-new
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化自动清理历史消息逻辑

Browse Source
This commit is contained in:
imhao183
2021-01-24 20:27:22 +08:00
parent 7923a3e0c4
commit 3d0a36b4f3
3 changed files with 160 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
broker/migrate/h2/V37__add_user_messages_mid_index.sql
Executable
+1
View File
@@ -0,0 +1 @@
alter table `t_user_messages` ADD INDEX `user_messages_mid_index` ( `_mid`);
broker/migrate/mysql/V42__add_user_messages_mid_index.sql
Executable
+129
View File
@@ -0,0 +1,129 @@
alter table `t_user_messages` add INDEX `user_messages_mid_index` ( `_mid` );
alter table `t_user_messages_0` add INDEX `user_messages0_mid_index` ( `_mid` );
alter table `t_user_messages_1` add INDEX `user_messages1_mid_index` ( `_mid` );
alter table `t_user_messages_2` add INDEX `user_messages2_mid_index` ( `_mid` );
alter table `t_user_messages_3` add INDEX `user_messages3_mid_index` ( `_mid` );
alter table `t_user_messages_4` add INDEX `user_messages4_mid_index` ( `_mid` );
alter table `t_user_messages_5` add INDEX `user_messages5_mid_index` ( `_mid` );
alter table `t_user_messages_6` add INDEX `user_messages6_mid_index` ( `_mid` );
alter table `t_user_messages_7` add INDEX `user_messages7_mid_index` ( `_mid` );
alter table `t_user_messages_8` add INDEX `user_messages8_mid_index` ( `_mid` );
alter table `t_user_messages_9` add INDEX `user_messages9_mid_index` ( `_mid` );
alter table `t_user_messages_10` add INDEX `user_messages10_mid_index` ( `_mid` );
alter table `t_user_messages_11` add INDEX `user_messages11_mid_index` ( `_mid` );
alter table `t_user_messages_12` add INDEX `user_messages12_mid_index` ( `_mid` );
alter table `t_user_messages_13` add INDEX `user_messages13_mid_index` ( `_mid` );
alter table `t_user_messages_14` add INDEX `user_messages14_mid_index` ( `_mid` );
alter table `t_user_messages_15` add INDEX `user_messages15_mid_index` ( `_mid` );
alter table `t_user_messages_16` add INDEX `user_messages16_mid_index` ( `_mid` );
alter table `t_user_messages_17` add INDEX `user_messages17_mid_index` ( `_mid` );
alter table `t_user_messages_18` add INDEX `user_messages18_mid_index` ( `_mid` );
alter table `t_user_messages_19` add INDEX `user_messages19_mid_index` ( `_mid` );
alter table `t_user_messages_20` add INDEX `user_messages20_mid_index` ( `_mid` );
alter table `t_user_messages_21` add INDEX `user_messages21_mid_index` ( `_mid` );
alter table `t_user_messages_22` add INDEX `user_messages22_mid_index` ( `_mid` );
alter table `t_user_messages_23` add INDEX `user_messages23_mid_index` ( `_mid` );
alter table `t_user_messages_24` add INDEX `user_messages24_mid_index` ( `_mid` );
alter table `t_user_messages_25` add INDEX `user_messages25_mid_index` ( `_mid` );
alter table `t_user_messages_26` add INDEX `user_messages26_mid_index` ( `_mid` );
alter table `t_user_messages_27` add INDEX `user_messages27_mid_index` ( `_mid` );
alter table `t_user_messages_28` add INDEX `user_messages28_mid_index` ( `_mid` );
alter table `t_user_messages_29` add INDEX `user_messages29_mid_index` ( `_mid` );
alter table `t_user_messages_30` add INDEX `user_messages30_mid_index` ( `_mid` );
alter table `t_user_messages_31` add INDEX `user_messages31_mid_index` ( `_mid` );
alter table `t_user_messages_32` add INDEX `user_messages32_mid_index` ( `_mid` );
alter table `t_user_messages_33` add INDEX `user_messages33_mid_index` ( `_mid` );
alter table `t_user_messages_34` add INDEX `user_messages34_mid_index` ( `_mid` );
alter table `t_user_messages_35` add INDEX `user_messages35_mid_index` ( `_mid` );
alter table `t_user_messages_36` add INDEX `user_messages36_mid_index` ( `_mid` );
alter table `t_user_messages_37` add INDEX `user_messages37_mid_index` ( `_mid` );
alter table `t_user_messages_38` add INDEX `user_messages38_mid_index` ( `_mid` );
alter table `t_user_messages_39` add INDEX `user_messages39_mid_index` ( `_mid` );
alter table `t_user_messages_40` add INDEX `user_messages40_mid_index` ( `_mid` );
alter table `t_user_messages_41` add INDEX `user_messages41_mid_index` ( `_mid` );
alter table `t_user_messages_42` add INDEX `user_messages42_mid_index` ( `_mid` );
alter table `t_user_messages_43` add INDEX `user_messages43_mid_index` ( `_mid` );
alter table `t_user_messages_44` add INDEX `user_messages44_mid_index` ( `_mid` );
alter table `t_user_messages_45` add INDEX `user_messages45_mid_index` ( `_mid` );
alter table `t_user_messages_46` add INDEX `user_messages46_mid_index` ( `_mid` );
alter table `t_user_messages_47` add INDEX `user_messages47_mid_index` ( `_mid` );
alter table `t_user_messages_48` add INDEX `user_messages48_mid_index` ( `_mid` );
alter table `t_user_messages_49` add INDEX `user_messages49_mid_index` ( `_mid` );
alter table `t_user_messages_50` add INDEX `user_messages50_mid_index` ( `_mid` );
alter table `t_user_messages_51` add INDEX `user_messages51_mid_index` ( `_mid` );
alter table `t_user_messages_52` add INDEX `user_messages52_mid_index` ( `_mid` );
alter table `t_user_messages_53` add INDEX `user_messages53_mid_index` ( `_mid` );
alter table `t_user_messages_54` add INDEX `user_messages54_mid_index` ( `_mid` );
alter table `t_user_messages_55` add INDEX `user_messages55_mid_index` ( `_mid` );
alter table `t_user_messages_56` add INDEX `user_messages56_mid_index` ( `_mid` );
alter table `t_user_messages_57` add INDEX `user_messages57_mid_index` ( `_mid` );
alter table `t_user_messages_58` add INDEX `user_messages58_mid_index` ( `_mid` );
alter table `t_user_messages_59` add INDEX `user_messages59_mid_index` ( `_mid` );
alter table `t_user_messages_60` add INDEX `user_messages60_mid_index` ( `_mid` );
alter table `t_user_messages_61` add INDEX `user_messages61_mid_index` ( `_mid` );
alter table `t_user_messages_62` add INDEX `user_messages62_mid_index` ( `_mid` );
alter table `t_user_messages_63` add INDEX `user_messages63_mid_index` ( `_mid` );
alter table `t_user_messages_64` add INDEX `user_messages64_mid_index` ( `_mid` );
alter table `t_user_messages_65` add INDEX `user_messages65_mid_index` ( `_mid` );
alter table `t_user_messages_66` add INDEX `user_messages66_mid_index` ( `_mid` );
alter table `t_user_messages_67` add INDEX `user_messages67_mid_index` ( `_mid` );
alter table `t_user_messages_68` add INDEX `user_messages68_mid_index` ( `_mid` );
alter table `t_user_messages_69` add INDEX `user_messages69_mid_index` ( `_mid` );
alter table `t_user_messages_70` add INDEX `user_messages70_mid_index` ( `_mid` );
alter table `t_user_messages_71` add INDEX `user_messages71_mid_index` ( `_mid` );
alter table `t_user_messages_72` add INDEX `user_messages72_mid_index` ( `_mid` );
alter table `t_user_messages_73` add INDEX `user_messages73_mid_index` ( `_mid` );
alter table `t_user_messages_74` add INDEX `user_messages74_mid_index` ( `_mid` );
alter table `t_user_messages_75` add INDEX `user_messages75_mid_index` ( `_mid` );
alter table `t_user_messages_76` add INDEX `user_messages76_mid_index` ( `_mid` );
alter table `t_user_messages_77` add INDEX `user_messages77_mid_index` ( `_mid` );
alter table `t_user_messages_78` add INDEX `user_messages78_mid_index` ( `_mid` );
alter table `t_user_messages_79` add INDEX `user_messages79_mid_index` ( `_mid` );
alter table `t_user_messages_80` add INDEX `user_messages80_mid_index` ( `_mid` );
alter table `t_user_messages_81` add INDEX `user_messages81_mid_index` ( `_mid` );
alter table `t_user_messages_82` add INDEX `user_messages82_mid_index` ( `_mid` );
alter table `t_user_messages_83` add INDEX `user_messages83_mid_index` ( `_mid` );
alter table `t_user_messages_84` add INDEX `user_messages84_mid_index` ( `_mid` );
alter table `t_user_messages_85` add INDEX `user_messages85_mid_index` ( `_mid` );
alter table `t_user_messages_86` add INDEX `user_messages86_mid_index` ( `_mid` );
alter table `t_user_messages_87` add INDEX `user_messages87_mid_index` ( `_mid` );
alter table `t_user_messages_88` add INDEX `user_messages88_mid_index` ( `_mid` );
alter table `t_user_messages_89` add INDEX `user_messages89_mid_index` ( `_mid` );
alter table `t_user_messages_90` add INDEX `user_messages90_mid_index` ( `_mid` );
alter table `t_user_messages_91` add INDEX `user_messages91_mid_index` ( `_mid` );
alter table `t_user_messages_92` add INDEX `user_messages92_mid_index` ( `_mid` );
alter table `t_user_messages_93` add INDEX `user_messages93_mid_index` ( `_mid` );
alter table `t_user_messages_94` add INDEX `user_messages94_mid_index` ( `_mid` );
alter table `t_user_messages_95` add INDEX `user_messages95_mid_index` ( `_mid` );
alter table `t_user_messages_96` add INDEX `user_messages96_mid_index` ( `_mid` );
alter table `t_user_messages_97` add INDEX `user_messages97_mid_index` ( `_mid` );
alter table `t_user_messages_98` add INDEX `user_messages98_mid_index` ( `_mid` );
alter table `t_user_messages_99` add INDEX `user_messages99_mid_index` ( `_mid` );
alter table `t_user_messages_100` add INDEX `user_messages100_mid_index` ( `_mid` );
alter table `t_user_messages_101` add INDEX `user_messages101_mid_index` ( `_mid` );
alter table `t_user_messages_102` add INDEX `user_messages102_mid_index` ( `_mid` );
alter table `t_user_messages_103` add INDEX `user_messages103_mid_index` ( `_mid` );
alter table `t_user_messages_104` add INDEX `user_messages104_mid_index` ( `_mid` );
alter table `t_user_messages_105` add INDEX `user_messages105_mid_index` ( `_mid` );
alter table `t_user_messages_106` add INDEX `user_messages106_mid_index` ( `_mid` );
alter table `t_user_messages_107` add INDEX `user_messages107_mid_index` ( `_mid` );
alter table `t_user_messages_108` add INDEX `user_messages108_mid_index` ( `_mid` );
alter table `t_user_messages_109` add INDEX `user_messages109_mid_index` ( `_mid` );
alter table `t_user_messages_110` add INDEX `user_messages110_mid_index` ( `_mid` );
alter table `t_user_messages_111` add INDEX `user_messages111_mid_index` ( `_mid` );
alter table `t_user_messages_112` add INDEX `user_messages112_mid_index` ( `_mid` );
alter table `t_user_messages_113` add INDEX `user_messages113_mid_index` ( `_mid` );
alter table `t_user_messages_114` add INDEX `user_messages114_mid_index` ( `_mid` );
alter table `t_user_messages_115` add INDEX `user_messages115_mid_index` ( `_mid` );
alter table `t_user_messages_116` add INDEX `user_messages116_mid_index` ( `_mid` );
alter table `t_user_messages_117` add INDEX `user_messages117_mid_index` ( `_mid` );
alter table `t_user_messages_118` add INDEX `user_messages118_mid_index` ( `_mid` );
alter table `t_user_messages_119` add INDEX `user_messages119_mid_index` ( `_mid` );
alter table `t_user_messages_120` add INDEX `user_messages120_mid_index` ( `_mid` );
alter table `t_user_messages_121` add INDEX `user_messages121_mid_index` ( `_mid` );
alter table `t_user_messages_122` add INDEX `user_messages122_mid_index` ( `_mid` );
alter table `t_user_messages_123` add INDEX `user_messages123_mid_index` ( `_mid` );
alter table `t_user_messages_124` add INDEX `user_messages124_mid_index` ( `_mid` );
alter table `t_user_messages_125` add INDEX `user_messages125_mid_index` ( `_mid` );
alter table `t_user_messages_126` add INDEX `user_messages126_mid_index` ( `_mid` );
alter table `t_user_messages_127` add INDEX `user_messages127_mid_index` ( `_mid` );
broker/src/main/java/win/liyufan/im/DBUtil.java
+30 -111
View File
@@ -166,133 +166,52 @@ public class DBUtil {
return SystemExiting;
}
private static void clearH2DB() {
boolean hasMore = true;
while (hasMore) {
hasMore = clearOneTable("t_user_messages");
if(sleep()) {
return;
}
}
hasMore = true;
while (hasMore) {
hasMore = clearOneTable("t_messages");
if(sleep()) {
return;
}
}
}
private static void clearMySQL() {
int i = 0;
do {
if(!clearOneTable("t_user_messages_" + i)) {
i++;
}
if(sleep()) {
return;
}
} while (i < 128);
boolean hasMore = true;
String msgTableName;
if(ClearDBDebugMode) {
msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis()));
} else {
msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis()), 1);
}
while (hasMore) {
hasMore = clearOneTable(msgTableName);
if(sleep()) {
return;
}
}
private static long getMaxMidNeedClean() {
Calendar cal = Calendar.getInstance();
cal.setTime(new Date());
if(cal.get(Calendar.DAY_OF_MONTH) > 25 && !ClearDBDebugMode) {
hasMore = true;
msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis()), 2);
while (hasMore) {
hasMore = clearOneTable(msgTableName);
if(sleep()) {
return;
}
}
}
cal.add(Calendar.YEAR, -3);
cal.add(Calendar.DATE, 1);
return MessageShardingUtil.getMsgIdFromTimestamp(cal.getTimeInMillis());
}
private static boolean clearOneTable(String tableName) {
private static void clearH2DB() {
clearOneTable("t_user_messages");
if(sleep()) {
return;
}
clearOneTable("t_messages");
}
private static void clearMySQL() {
for (int i = 0; i < 128; i++) {
clearOneTable("t_user_messages_" + i);
if(sleep()) {
return;
}
}
String msgTableName = MessageShardingUtil.getMessageTable(MessageShardingUtil.getMsgIdFromTimestamp(System.currentTimeMillis()));
clearOneTable(msgTableName);
}
private static void clearOneTable(String tableName) {
Connection connection = null;
PreparedStatement statement = null;
ResultSet rs = null;
int clearCount = 999;
if(ClearDBDebugMode) {
clearCount = 19;
}
boolean hasMore = true;
long mid = getMaxMidNeedClean();
try {
connection = DBUtil.getConnection();
String sql = "select id, _mid from " + tableName + " order by id limit 1 offset " + clearCount;
String sql = "delete from " + tableName + " where _mid <= " + mid;
statement = connection.prepareStatement(sql);
rs = statement.executeQuery();
int id = 0;
long mid = 0;
if (rs.next()) {
id = rs.getInt(1);
mid = rs.getLong(2);
}
if(id == 0) {
hasMore = false;
rs.close();
rs = null;
statement.close();
sql = "select id, _mid from " + tableName + " order by id desc limit 1";
statement = connection.prepareStatement(sql);
rs = statement.executeQuery();
if (rs.next()) {
id = rs.getInt(1);
mid = rs.getLong(2);
}
}
rs.close();
rs = null;
LOG.info("Check history message in table {}, id {}, mid {}, hasMore {}", tableName, id, mid, hasMore);
if(id != 0) {
Calendar tm = MessageShardingUtil.getCalendarFromMessageId(mid);
if(ClearDBDebugMode) {
tm.add(Calendar.HOUR, 1);
} else {
tm.add(Calendar.YEAR, 3);
tm.add(Calendar.MONTH, -1);
tm.add(Calendar.DATE, -4);
}
if(tm.getTimeInMillis() < System.currentTimeMillis()) {
statement.close();
sql = "delete from " + tableName + " where id <= " + id;
statement = connection.prepareStatement(sql);
int count = statement.executeUpdate();
LOG.info("Clear {} rows", count);
} else {
LOG.info("No history messages need clean");
hasMore = false;
}
} else {
LOG.info("No messages");
}
int count = statement.executeUpdate();
LOG.info("Delete history message {} rows", count);
} catch (Exception e) {
hasMore = false;
e.printStackTrace();
Utility.printExecption(LOG, e, RDBS_Exception);
} finally {
DBUtil.closeDB(connection, statement, rs);
}
return hasMore;
}
private static List<String> getCreateSql() {