anna/IM-server-new
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduced the FlightZone concept (inbound and outbound) and separated logic of storage system.

Browse Source
This commit is contained in:
andsel
2017-04-09 19:22:50 +02:00
parent 5c6889fbd5
commit 6dc762f47f
21 changed files with 356 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mapdb_storage/src/main/java/io/moquette/persistence/mapdb/MapDBMessagesStore.java
+13 -53
View File
@@ -48,14 +48,14 @@ class MapDBMessagesStore implements IMessagesStore {
@Override
public void initStore() {
LOG.info("Initializing store...");
m_retainedStore = m_db.getHashMap("retained");
m_persistentMessageStore = m_db.getHashMap("persistedMessages");
LOG.info("Initialized store");
}
@Override
public void storeRetained(Topic topic, MessageGUID guid) {
LOG.debug("Storing retained messages. Topic = {}, guid = {}.", topic, guid);
LOG.debug("Storing retained messages. Topic={}, guid={}", topic, guid);
m_retainedStore.put(topic, guid);
}
@@ -72,7 +72,7 @@ class MapDBMessagesStore implements IMessagesStore {
}
if (LOG.isTraceEnabled()) {
LOG.trace("The retained messages have been scanned. MatchingMessages = {}.", results);
LOG.trace("The retained messages have been scanned. MatchingMessages={}", results);
}
return results;
@@ -80,75 +80,35 @@ class MapDBMessagesStore implements IMessagesStore {
@Override
public MessageGUID storePublishForFuture(StoredMessage storedMessage) {
assert storedMessage.getClientID() != null : "The message to be persisted must have a valid client ID";
assert storedMessage.getClientID() != null : "Message to be persisted must have a valid client ID";
MessageGUID guid = new MessageGUID(UUID.randomUUID().toString());
storedMessage.setGuid(guid);
LOG.debug(
"Storing publish event. MqttClientId = {}, messageId = {}, guid = {}, topic = {}.",
storedMessage.getClientID(),
storedMessage.getMessageID(),
guid,
storedMessage.getTopic());
LOG.debug("Storing publish event. CId={}, guid={}, topic={}", storedMessage.getClientID(), guid,
storedMessage.getTopic());
m_persistentMessageStore.put(guid, storedMessage);
ConcurrentMap<Integer, MessageGUID> messageIdToGuid = m_db
.getHashMap(messageId2GuidsMapName(storedMessage.getClientID()));
messageIdToGuid.put(storedMessage.getMessageID(), guid);
return guid;
}
@Override
public void dropMessagesInSession(String clientID) {
LOG.debug("Dropping stored messages. ClientId = {}.", clientID);
ConcurrentMap<Integer, MessageGUID> messageIdToGuid = m_db.getHashMap(messageId2GuidsMapName(clientID));
for (MessageGUID guid : messageIdToGuid.values()) {
removeStoredMessage(guid);
}
messageIdToGuid.clear();
}
public void dropInFlightMessagesInSession(Collection<MessageGUID> pendingAckMessages) {
//remove all guids from retained
Collection<MessageGUID> messagesToRemove = new HashSet<>(pendingAckMessages);
messagesToRemove.removeAll(m_retainedStore.values());
void removeStoredMessage(MessageGUID guid) {
// remove only the not retained and no more referenced
StoredMessage storedMessage = m_persistentMessageStore.get(guid);
if (!storedMessage.isRetained()) {
LOG.debug(
"Dropping stored message. ClientId = {}, messageId = {}, guid = {}, topic = {}.",
storedMessage.getClientID(),
storedMessage.getMessageID(),
guid,
storedMessage.getTopic());
for (MessageGUID guid : messagesToRemove) {
m_persistentMessageStore.remove(guid);
}
}
@Override
public StoredMessage getMessageByGuid(MessageGUID guid) {
LOG.debug("Retrieving stored message. Guid = {}.", guid);
LOG.debug("Retrieving stored message. Guid={}", guid);
return m_persistentMessageStore.get(guid);
}
@Override
public void cleanRetained(Topic topic) {
LOG.debug("Cleaning retained messages. Topic = {}.", topic);
LOG.debug("Cleaning retained messages. Topic={}", topic);
m_retainedStore.remove(topic);
}
@Override
public int getPendingPublishMessages(String clientID) {
ConcurrentMap<Integer, MessageGUID> messageIdToGuidMap = m_db
.getHashMap(messageId2GuidsMapName(clientID));
return messageIdToGuidMap.size();
}
@Override
public MessageGUID mapToGuid(String clientID, int messageID) {
LOG.debug("Mapping message ID to GUID CId={}, messageId={}", clientID, messageID);
ConcurrentMap<Integer, MessageGUID> messageIdToGuid = m_db.getHashMap(messageId2GuidsMapName(clientID));
MessageGUID result = messageIdToGuid.get(messageID);
LOG.debug("Message ID has been mapped to a GUID CId={}, messageId={}, guid={}", clientID, messageID, result);
return result;
}
static String messageId2GuidsMapName(String clientID) {
return "guidsMapping_" + clientID;
}
}
mapdb_storage/src/main/java/io/moquette/persistence/mapdb/MapDBSessionsStore.java
+35 -1
View File
@@ -28,6 +28,7 @@ import io.moquette.spi.impl.subscriptions.Topic;
import org.mapdb.DB;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
@@ -310,9 +311,24 @@ class MapDBSessionsStore implements ISessionsStore {
return m_inflightStore.get(clientID).size();
}
@Override
public StoredMessage inboundInflight(String clientID, int messageID) {
LOG.debug("Mapping inbound message ID to GUID CId={}, messageId={}", clientID, messageID);
ConcurrentMap<Integer, MessageGUID> messageIdToGuid = m_db.getHashMap(inboundMessageId2GuidsMapName(clientID));
final MessageGUID guid = messageIdToGuid.get(messageID);
LOG.debug("Inbound message ID has been mapped CId={}, messageId={}, guid={}", clientID, messageID, guid);
return m_messagesStore.getMessageByGuid(guid);
}
@Override
public void markAsInboundInflight(String clientID, int messageID, MessageGUID guid) {
ConcurrentMap<Integer, MessageGUID> messageIdToGuid = m_db.getHashMap(inboundMessageId2GuidsMapName(clientID));
messageIdToGuid.put(messageID, guid);
}
@Override
public int getPendingPublishMessagesNo(String clientID) {
return m_messagesStore.getPendingPublishMessages(clientID);
return queue(clientID).size();
}
@Override
@@ -322,4 +338,22 @@ class MapDBSessionsStore implements ISessionsStore {
else
return m_secondPhaseStore.get(clientID).size();
}
@Override
public Collection<MessageGUID> pendingAck(String clientID) {
ConcurrentMap<Integer, MessageGUID> messageGUIDMap = m_db.getHashMap(messageId2GuidsMapName(clientID));
if (messageGUIDMap == null || messageGUIDMap.isEmpty()) {
return Collections.emptyList();
}
return new ArrayList<>(messageGUIDMap.values());
}
static String messageId2GuidsMapName(String clientID) {
return "guidsMapping_" + clientID;
}
static String inboundMessageId2GuidsMapName(String clientID) {
return "inboundInflightGuidsMapping_" + clientID;
}
}
mapdb_storage/src/test/java/io/moquette/persistence/mapdb/MapDBPersistentStoreTest.java
+17 -24
View File
@@ -16,9 +16,6 @@
package io.moquette.persistence.mapdb;
import java.io.File;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import io.moquette.BrokerConstants;
import io.moquette.server.config.IConfig;
import io.moquette.server.config.MemoryConfig;
@@ -29,15 +26,19 @@ import io.moquette.spi.ISessionsStore.ClientTopicCouple;
import io.moquette.spi.MessageGUID;
import io.moquette.spi.impl.subscriptions.Subscription;
import io.moquette.spi.impl.subscriptions.Topic;
import io.netty.handler.codec.mqtt.MqttQoS;
import static io.netty.handler.codec.mqtt.MqttQoS.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static io.moquette.BrokerConstants.PERSISTENT_STORE_PROPERTY_NAME;
import static java.util.Collections.singletonList;
import static org.junit.Assert.*;
/**
@@ -92,14 +93,11 @@ public class MapDBPersistentStoreTest {
ClientSession session1 = m_sessionsStore.createNewSession("SESSION_ID_1", true);
// Subscribe on /topic with QOSType.MOST_ONE
Subscription oldSubscription = new Subscription(session1.clientID, new Topic("/topic"), MqttQoS.AT_MOST_ONCE);
Subscription oldSubscription = new Subscription(session1.clientID, new Topic("/topic"), AT_MOST_ONCE);
session1.subscribe(oldSubscription);
// Subscribe on /topic again that overrides the previous subscription.
Subscription overridingSubscription = new Subscription(
session1.clientID,
new Topic("/topic"),
MqttQoS.EXACTLY_ONCE);
Subscription overridingSubscription = new Subscription(session1.clientID, new Topic("/topic"), EXACTLY_ONCE);
session1.subscribe(overridingSubscription);
// Verify
@@ -156,18 +154,15 @@ public class MapDBPersistentStoreTest {
@Test
public void testDropMessagesInSessionCleanAllNotRetainedStoredMessages() {
m_sessionsStore.createNewSession("TestClient", true);
IMessagesStore.StoredMessage publishToStore = new IMessagesStore.StoredMessage(
"Hello".getBytes(),
MqttQoS.EXACTLY_ONCE,
"/topic");
m_sessionsStore.createNewSession(TEST_CLIENT, true);
IMessagesStore.StoredMessage publishToStore = new IMessagesStore.StoredMessage("Hello".getBytes(), EXACTLY_ONCE,
"/topic");
publishToStore.setClientID(TEST_CLIENT);
publishToStore.setMessageID(1);
publishToStore.setRetained(false);
MessageGUID guid = m_messagesStore.storePublishForFuture(publishToStore);
// Exercise
m_messagesStore.dropMessagesInSession("TestClient");
m_messagesStore.dropInFlightMessagesInSession(singletonList(guid));
// Verify the message store for session is empty.
IMessagesStore.StoredMessage storedPublish = m_messagesStore.getMessageByGuid(guid);
@@ -176,18 +171,16 @@ public class MapDBPersistentStoreTest {
@Test
public void testDropMessagesInSessionDoesntCleanAnyRetainedStoredMessages() {
m_sessionsStore.createNewSession("TestClient", true);
IMessagesStore.StoredMessage publishToStore = new IMessagesStore.StoredMessage(
"Hello".getBytes(),
MqttQoS.EXACTLY_ONCE,
"/topic");
m_sessionsStore.createNewSession(TEST_CLIENT, true);
IMessagesStore.StoredMessage publishToStore = new IMessagesStore.StoredMessage("Hello".getBytes(), EXACTLY_ONCE,
"/topic");
publishToStore.setClientID(TEST_CLIENT);
publishToStore.setMessageID(1);
publishToStore.setRetained(true);
MessageGUID guid = m_messagesStore.storePublishForFuture(publishToStore);
m_messagesStore.storeRetained(new Topic("/topic"), guid);
// Exercise
m_messagesStore.dropMessagesInSession("TestClient");
m_messagesStore.dropInFlightMessagesInSession(singletonList(guid));
// Verify the message store for session is empty.
IMessagesStore.StoredMessage storedPublish = m_messagesStore.getMessageByGuid(guid);