From 2bafce379bda1336082f5f108dd73c233498725c Mon Sep 17 00:00:00 2001 From: Saurav Das <saurav.das@bigswitch.com> Date: Wed, 22 May 2013 14:04:36 -0700 Subject: [PATCH] Changing DebugCounter implementation to be more like DebugEvents in terms of performace of updates. Introducing hierarchical counters, counter metatdata and finer-grained handling of counter flush. Added unit tests and changed some of the API calls --- .../core/OFSwitchBase.java | 42 +- .../core/internal/Controller.java | 306 +++++---- .../core/internal/OFChannelHandler.java | 37 +- .../debugcounter/DebugCounter.java | 638 +++++++++++++----- .../debugcounter/IDebugCounterService.java | 53 +- .../debugcounter/NullDebugCounter.java | 59 +- .../web/DebugCounterResource.java | 252 ++++--- .../web/DebugCounterRoutable.java | 12 +- .../debugevent/DebugEvent.java | 2 +- .../debugevent/IDebugEventService.java | 2 +- .../internal/DeviceManagerImpl.java | 417 ++++++------ .../internal/LinkDiscoveryManager.java | 52 +- .../topology/TopologyManager.java | 17 +- .../sync/internal/SyncManager.java | 257 +++---- .../sync/internal/rpc/RPCChannelHandler.java | 182 ++--- .../internal/store/ListenerStorageEngine.java | 16 +- .../CounterHierarchyBelowTest.java | 169 +++++ .../debugcounter/CounterHierarchyGetTest.java | 278 ++++++++ .../debugcounter/CounterHierarchyPutTest.java | 121 ++++ 19 files changed, 1949 insertions(+), 963 deletions(-) create mode 100644 src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyBelowTest.java create mode 100644 src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyGetTest.java create mode 100644 src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyPutTest.java diff --git a/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java b/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java index ac75c6b3e..bfecf6992 100644 --- a/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java +++ b/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java @@ -44,6 +44,7 @@ import net.floodlightcontroller.core.util.AppCookie; import net.floodlightcontroller.core.web.serializers.DPIDSerializer; import net.floodlightcontroller.debugcounter.IDebugCounterService; import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType; +import net.floodlightcontroller.debugcounter.IDebugCounterService.MaxCountersRegistered; import net.floodlightcontroller.debugcounter.NullDebugCounter; import net.floodlightcontroller.devicemanager.SwitchPort; import net.floodlightcontroller.packet.Ethernet; @@ -139,6 +140,10 @@ public abstract class OFSwitchBase implements IOFSwitch { protected OFDescriptionStatistics description; private boolean debugCountersRegistered; + @SuppressWarnings("unused") + private int SWITCH, SWITCH_PKTIN, SWITCH_WRITE; + private int SWITCH_PKTIN_DROPS, SWITCH_WRITE_DROPS; + protected final static ThreadLocal<Map<IOFSwitch,List<OFMessage>>> local_msg_buffer = new ThreadLocal<Map<IOFSwitch,List<OFMessage>>>() { @@ -726,7 +731,7 @@ public abstract class OFSwitchBase implements IOFSwitch { write(m, bc); } else { // Let logback duplicate filtering take care of excessive logs - debugCounters.updateCounter(stringId + "-writeDrops"); + debugCounters.updateCounter(SWITCH_WRITE_DROPS, false); log.warn("Drop throttled OF message to switch {}", this); } } @@ -738,8 +743,7 @@ public abstract class OFSwitchBase implements IOFSwitch { write(msglist, bc); } else { // Let logback duplicate filtering take care of excessive logs - debugCounters.updateCounter(stringId + "-writeDrops", - msglist.size()); + debugCounters.updateCounter(SWITCH_WRITE_DROPS, msglist.size(), false); log.warn("Drop throttled OF messages to switch {}", this); } } @@ -1330,7 +1334,7 @@ public abstract class OFSwitchBase implements IOFSwitch { OFMatch match = new OFMatch(); match.loadFromPacket(pin.getPacketData(), pin.getInPort()); if (ofMatchCache.update(match)) { - debugCounters.updateCounter(stringId + "-pktinDrops"); + debugCounters.updateCounter(SWITCH_PKTIN_DROPS, false); return true; } @@ -1390,10 +1394,32 @@ public abstract class OFSwitchBase implements IOFSwitch { debugCountersRegistered = true; return; } - debugCounters.registerCounter(stringId + "-pktinDrops", - "Packet in throttle drop count", CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(stringId + "-writeDrops", - "Switch write throttle drop count", CounterType.ALWAYS_COUNT); + try { + // every level of the hierarchical counter has to be registered + // even if they are not used + SWITCH = debugCounters.registerCounter( + "switch", stringId, + "Counter for this switch", + CounterType.ALWAYS_COUNT, new Object[] {}); + SWITCH_PKTIN = debugCounters.registerCounter( + "switch", stringId + "/pktin", + "Packet in counter for this switch", + CounterType.ALWAYS_COUNT, new Object[] {}); + SWITCH_WRITE = debugCounters.registerCounter( + "switch", stringId + "/write", + "Write counter for this switch", + CounterType.ALWAYS_COUNT, new Object[] {}); + SWITCH_PKTIN_DROPS = debugCounters.registerCounter( + "switch", stringId + "/pktin/drops", + "Packet in throttle drop count", + CounterType.ALWAYS_COUNT, new Object[] {}); + SWITCH_WRITE_DROPS = debugCounters.registerCounter( + "switch", stringId + "/write/drops", + "Switch write throttle drop count", + CounterType.ALWAYS_COUNT, new Object[] {}); + } catch (MaxCountersRegistered e) { + e.printStackTrace(); + } } /** diff --git a/src/main/java/net/floodlightcontroller/core/internal/Controller.java b/src/main/java/net/floodlightcontroller/core/internal/Controller.java index e759317aa..568dd3d20 100644 --- a/src/main/java/net/floodlightcontroller/core/internal/Controller.java +++ b/src/main/java/net/floodlightcontroller/core/internal/Controller.java @@ -72,6 +72,7 @@ import net.floodlightcontroller.core.web.CoreWebRoutable; import net.floodlightcontroller.counter.ICounterStoreService; import net.floodlightcontroller.debugcounter.IDebugCounterService; import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType; +import net.floodlightcontroller.debugcounter.IDebugCounterService.MaxCountersRegistered; import net.floodlightcontroller.debugevent.IDebugEventService; import net.floodlightcontroller.debugevent.NullDebugEvent; import net.floodlightcontroller.debugevent.IDebugEventService.EventType; @@ -229,29 +230,33 @@ public class Controller implements IFloodlightProviderService, */ public static class Counter { private final IDebugCounterService debugCounterService; - private final String name; + private int counterId; public Counter(IDebugCounterService debugCounterService, - String name, + String moduleName, String counterName, String description, - CounterType type) { + CounterType type, Object[] metaData) { this.debugCounterService = debugCounterService; - this.name = name; - this.debugCounterService.registerCounter(name, - description, - type); + try { + this.counterId = this.debugCounterService.registerCounter( + moduleName, counterName, description, + type, metaData); + } catch (MaxCountersRegistered e) { + e.printStackTrace(); + } } - public void increment() { - this.debugCounterService.updateCounter(name); + public void increment(boolean flushNow) { + this.debugCounterService.updateCounter(this.counterId, flushNow); } - public void increment(int incrementAmount) { - this.debugCounterService.updateCounter(name, incrementAmount); + public void increment(int incrementAmount, boolean flushNow) { + this.debugCounterService.updateCounter(this.counterId, incrementAmount, + flushNow); } } public static class Counters { - public static final String prefix = "controller-"; + public static final String prefix = "controller"; public Counter setRoleEqual; public Counter setSameRole; public Counter setRoleMaster; @@ -299,174 +304,170 @@ public class Controller implements IFloodlightProviderService, public Counter roleReplyReceived; // expected RoleReply received public Counter roleReplyErrorUnsupported; - private IDebugCounterService debugCounters = null; - void flushCounters() { - if (debugCounters != null) - debugCounters.flushCounters(); - } + private static final String WARN = "warn"; + private static final String ERROR = "error"; void createCounters(IDebugCounterService debugCounters) { - this.debugCounters = debugCounters; setRoleEqual = new Counter(debugCounters, - prefix + "setRoleEqual", + prefix, "setRoleEqual", "Controller received a role request with role of "+ "EQUAL which is unusual", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); setSameRole = new Counter(debugCounters, - prefix + "setSameRole", + prefix, "setSameRole", "Controller received a role request for the same " + "role the controller already had", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); setRoleMaster = new Counter(debugCounters, - prefix + "setRoleMaster", + prefix, "setRoleMaster", "Controller received a role request with role of " + "MASTER. This counter can be at most 1.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); remoteStoreNotification = new Counter(debugCounters, - prefix + "remoteStoreNotification", + prefix, "remoteStoreNotification", "Received a notification from the sync service " + "indicating that switch information has changed", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); invalidPortsChanged = new Counter(debugCounters, - prefix + "invalidPortsChanged", + prefix, "invalidPortsChanged", "Received an unexpected ports changed " + "notification while the controller was in " + "SLAVE role.", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); invalidSwitchActivatedWhileSlave = new Counter(debugCounters, - prefix + "invalidSwitchActivatedWhileSlave", + prefix, "invalidSwitchActivatedWhileSlave", "Received an unexpected switchActivated " + "notification while the controller was in " + "SLAVE role.", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); invalidStoreEventWhileMaster = new Counter(debugCounters, - prefix + "invalidSToreEventWhileMaster", + prefix, "invalidSToreEventWhileMaster", "Received an unexpected notification from " + "the sync store while the controller was in " + "MASTER role.", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); switchDisconnectedWhileSlave = new Counter(debugCounters, - prefix + "switchDisconnectedWhileSlave", + prefix, "switchDisconnectedWhileSlave", "A switch disconnected and the controller was " + "in SLAVE role.", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); switchActivated = new Counter(debugCounters, - prefix + "switchActivated", + prefix, "switchActivated", "A switch connected to this controller is now " + "in MASTER role", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); errorSameSwitchReactivated = // err new Counter(debugCounters, - prefix + "errorSameSwitchReactivated", + prefix, "errorSameSwitchReactivated", "A switch that was already in active state " + "was activated again. This indicates a " + "controller defect", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchWithSameDpidActivated = // warn new Counter(debugCounters, - prefix + "switchWithSameDpidActivated", + prefix, "switchWithSameDpidActivated", "A switch with the same DPID as another switch " + "connected to the controller. This can be " + "caused by multiple switches configured with " + "the same DPID or by a switch reconnecting very " + "quickly.", - CounterType.WARN); + CounterType.COUNT_ON_DEMAND, new Object[] {WARN}); newSwitchActivated = // new switch new Counter(debugCounters, - prefix + "newSwitchActivated", + prefix, "newSwitchActivated", "A new switch has completed the handshake as " + "MASTER. The switch was not known to any other " + "controller in the cluster", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); syncedSwitchActivated = new Counter(debugCounters, - prefix + "syncedSwitchActivated", + prefix, "syncedSwitchActivated", "A switch has completed the handshake as " + "MASTER. The switch was known to another " + "controller in the cluster", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); readyForReconcile = new Counter(debugCounters, - prefix + "readyForReconcile", + prefix, "readyForReconcile", "Controller is ready for flow reconciliation " + "after Slave to Master transition. Either all " + "previously known switches are now active " + "or they have timed out and have been removed." + "This counter will be 0 or 1.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); newSwitchFromStore = new Counter(debugCounters, - prefix + "newSwitchFromStore", + prefix, "newSwitchFromStore", "A new switch has connected to another " + "another controller in the cluster. This " + "controller instance has received a sync store " + "notification for it.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); updatedSwitchFromStore = new Counter(debugCounters, - prefix + "updatedSwitchFromStore", + prefix, "updatedSwitchFromStore", "Information about a switch connected to " + "another controller instance was updated in " + "the sync store. This controller instance has " + "received a notification for it", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); switchDisconnected = new Counter(debugCounters, - prefix + "switchDisconnected", + prefix, "switchDisconnected", "FIXME: switch has disconnected", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); syncedSwitchRemoved = new Counter(debugCounters, - prefix + "syncedSwitchRemoved", + prefix, "syncedSwitchRemoved", "A switch connected to another controller " + "instance has disconnected from the controller " + "cluster. This controller instance has " + "received a notification for it", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); unknownSwitchRemovedFromStore = new Counter(debugCounters, - prefix + "unknownSwitchRemovedFromStore", + prefix, "unknownSwitchRemovedFromStore", "This controller instances has received a sync " + "store notification that a switch has " + "disconnected but this controller instance " + "did not have the any information about the " + - "switch", - CounterType.WARN); // might be less than warning + "switch", // might be less than warning + CounterType.ALWAYS_COUNT, new Object[] {WARN}); consolidateStoreRunCount = new Counter(debugCounters, - prefix + "consolidateStoreRunCount", + prefix, "consolidateStoreRunCount", "This controller has transitioned from SLAVE " + "to MASTER and waited for switches to reconnect. " + "The controller has finished waiting and has " + "reconciled switch entries in the sync store " + "with live state", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); consolidateStoreInconsistencies = new Counter(debugCounters, - prefix + "consolidateStoreInconsistencies", + prefix, "consolidateStoreInconsistencies", "During switch sync store consolidation: " + "Number of switches that were in the store " + "but not otherwise known plus number of " + @@ -477,186 +478,187 @@ public class Controller implements IFloodlightProviderService, "written them. A non-zero count " + "indicates a brief split-brain dual MASTER " + "situation during fail-over", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {}); storeSyncError = new Counter(debugCounters, - prefix + "storeSyncError", + prefix, "storeSyncError", "Number of times a sync store operation failed " + "due to a store sync exception or an entry in " + "in the store had invalid data.", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchesNotReconnectingToNewMaster = new Counter(debugCounters, - prefix + "switchesNotReconnectingToNewMaster", + prefix, "switchesNotReconnectingToNewMaster", "Switches that were connected to another " + "controller instance in the cluster but that " + "did not reconnect to this controller after it " + - "transitioned to MASTER", - CounterType.WARN); // might be less than warning + "transitioned to MASTER", // might be less than warning + CounterType.ALWAYS_COUNT, new Object[] {}); switchPortChanged = new Counter(debugCounters, - prefix + "switchPortChanged", + prefix, "switchPortChanged", "Number of times switch ports have changed", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); switchOtherChange = new Counter(debugCounters, - prefix + "switchOtherChange", + prefix, "switchOtherChange", "Number of times other information of a switch " + "has changed.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); dispatchMessageWhileSlave = new Counter(debugCounters, - prefix + "dispatchMessageWhileSlave", + prefix, "dispatchMessageWhileSlave", "Number of times an OF message was received " + "and supposed to be dispatched but the " + "controller was in SLAVE role and the message " + "was not dispatched", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); dispatchMessage = // does this cnt make sense? more specific?? per type? count stops? new Counter(debugCounters, - prefix + "dispatchMessage", + prefix, "dispatchMessage", "Number of times an OF message was dispatched " + "to registered modules", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); controllerNodeIpsChanged = new Counter(debugCounters, - prefix + "controllerNodesIpsChanged", + prefix, "controllerNodesIpsChanged", "IP addresses of controller nodes have changed", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); //------------------------ // channel handler counters. Factor them out ?? messageReceived = new Counter(debugCounters, - prefix + "messageReceived", + prefix, "messageReceived", "Number of OpenFlow messages received. Some of " + "these might be throttled", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); messageInputThrottled = new Counter(debugCounters, - prefix + "messageInputThrottled", + prefix, "messageInputThrottled", "Number of OpenFlow messages that were " + "throttled due to high load from the sender", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); // TODO: more counters in messageReceived ?? switchDisconnectReadTimeout = new Counter(debugCounters, - prefix + "switchDisconnectReadTimeout", + prefix, "switchDisconnectReadTimeout", "Number of times a switch was disconnected due " + "due the switch failing to send OpenFlow " + "messages or responding to OpenFlow ECHOs", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchDisconnectHandshakeTimeout = new Counter(debugCounters, - prefix + "switchDisconnectHandshakeTimeout", + prefix, "switchDisconnectHandshakeTimeout", "Number of times a switch was disconnected " + "because it failed to complete the handshake " + "in time.", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchDisconnectIOError = new Counter(debugCounters, - prefix + "switchDisconnectIOError", + prefix, "switchDisconnectIOError", "Number of times a switch was disconnected " + "due to IO errors on the switch connection.", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchDisconnectParseError = new Counter(debugCounters, - prefix + "switchDisconnectParseError", + prefix, "switchDisconnectParseError", "Number of times a switch was disconnected " + "because it sent an invalid packet that could " + "not be parsed", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchDisconnectSwitchStateException = new Counter(debugCounters, - prefix + "switchDisconnectSwitchStateException", + prefix, "switchDisconnectSwitchStateException", "Number of times a switch was disconnected " + "because it sent messages that were invalid " + "given the switch connection's state.", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); rejectedExecutionException = new Counter(debugCounters, - prefix + "rejectedExecutionException", + prefix, "rejectedExecutionException", "TODO", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchDisconnectOtherException = new Counter(debugCounters, - prefix + "switchDisconnectOtherException", + prefix, "switchDisconnectOtherException", "Number of times a switch was disconnected " + "due to an exceptional situation not covered " + "by other counters", - CounterType.ERROR); + CounterType.ALWAYS_COUNT, new Object[] {ERROR}); switchConnected = new Counter(debugCounters, - prefix + "switchConnected", + prefix, "switchConnected", "Number of times a new switch connection was " + "established", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); unhandledMessage = new Counter(debugCounters, - prefix + "unhandledMessage", + prefix, "unhandledMessage", "Number of times an OpenFlow message was " + "received that the controller ignored because " + "it was inapproriate given the switch " + "connection's state.", - CounterType.WARN); // might be less than warning + CounterType.ALWAYS_COUNT, new Object[] {WARN}); + // might be less than warning packetInWhileSwitchIsSlave = new Counter(debugCounters, - prefix + "packetInWhileSwitchIsSlave", + prefix, "packetInWhileSwitchIsSlave", "Number of times a packet in was received " + "from a switch that was in SLAVE role. " + "Possibly inidicates inconsistent roles.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); epermErrorWhileSwitchIsMaster = new Counter(debugCounters, - prefix + "epermErrorWhileSwitchIsMaster", + prefix, "epermErrorWhileSwitchIsMaster", "Number of times a permission error was " + "received while the switch was in MASTER role. " + "Possibly inidicates inconsistent roles.", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); roleNotResentBecauseRolePending = new Counter(debugCounters, - prefix + "roleNotResentBecauseRolePending", + prefix, "roleNotResentBecauseRolePending", "The controller tried to reestablish a role " + "with a switch but did not do so because a " + "previous role request was still pending", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); roleRequestSent = new Counter(debugCounters, - prefix + "roleRequestSent", + prefix, "roleRequestSent", "Number of times the controller sent a role " + "request to a switch.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); roleReplyTimeout = new Counter(debugCounters, - prefix + "roleReplyTimeout", + prefix, "roleReplyTimeout", "Number of times a role request message did not " + "receive the expected reply from a switch", - CounterType.WARN); + CounterType.ALWAYS_COUNT, new Object[] {WARN}); roleReplyReceived = // expected RoleReply received new Counter(debugCounters, - prefix + "roleReplyReceived", + prefix, "roleReplyReceived", "Number of times the controller received the " + "expected role reply message from a switch", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); roleReplyErrorUnsupported = new Counter(debugCounters, - prefix + "roleReplyErrorUnsupported", + prefix, "roleReplyErrorUnsupported", "Number of times the controller received an " + "error from a switch in response to a role " + "request indicating that the switch does not " + "support roles.", - CounterType.ALWAYS_COUNT); + CounterType.ALWAYS_COUNT, new Object[] {}); } } @@ -783,13 +785,13 @@ public class Controller implements IFloodlightProviderService, "not be null"); } if (role == Role.EQUAL) { - counters.setRoleEqual.increment(); + counters.setRoleEqual.increment(true); log.debug("Received role request for EQUAL, setting to MASTER" + " instead"); role = Role.MASTER; } if (role == this.role) { - counters.setSameRole.increment(); + counters.setSameRole.increment(true); log.debug("Received role request for {} but controller is " + "already {}. Ignoring it.", role, this.role); return; @@ -804,7 +806,7 @@ public class Controller implements IFloodlightProviderService, // At this point we are guaranteed that we will execute the code // below exactly once during the lifetime of this process! And // it will be a to MASTER transition - counters.setRoleMaster.increment(); + counters.setRoleMaster.increment(true); log.info("Received role request for {} (reason: {})." + " Initiating transition", role, roleChangeDescription); @@ -864,14 +866,14 @@ public class Controller implements IFloodlightProviderService, // We only care for remote updates return; } - counters.remoteStoreNotification.increment(); + counters.remoteStoreNotification.increment(true); while(keys.hasNext()) { Long key = keys.next(); Versioned<SwitchSyncRepresentation> versionedSwitch = null; try { versionedSwitch = storeClient.get(key); } catch (SyncException e) { - counters.storeSyncError.increment(); + counters.storeSyncError.increment(true); log.error("Exception while retrieving switch " + HexString.toHexString(key) + " from sync store. Skipping", e); @@ -892,7 +894,7 @@ public class Controller implements IFloodlightProviderService, IOFSwitch sw = getOFSwitchInstance(storedSwitch.getDescription()); sw.setFeaturesReply(storedSwitch.getFeaturesReply()); if (!key.equals(storedSwitch.getFeaturesReply().getDatapathId())) { - counters.storeSyncError.increment(); + counters.storeSyncError.increment(true); log.error("Inconsistent DPIDs from switch sync store: " + "key is {} but sw.getId() says {}. Ignoring", HexString.toHexString(key), sw.getStringId()); @@ -909,7 +911,6 @@ public class Controller implements IFloodlightProviderService, @Override public void run() { consolidateStore(); - debugCounters.flushCounters(); } }; Controller.this.ses.schedule(consolidateStoreTask, @@ -945,12 +946,12 @@ public class Controller implements IFloodlightProviderService, */ public synchronized void switchActivated(IOFSwitch sw) { if (role != Role.MASTER) { - counters.invalidSwitchActivatedWhileSlave.increment(); + counters.invalidSwitchActivatedWhileSlave.increment(true); return; // only react to switch connections when master // FIXME: should we disconnect the switch? When can this happen? } Long dpid = sw.getId(); - counters.switchActivated.increment(); + counters.switchActivated.increment(true); IOFSwitch oldSw = this.activeSwitches.put(dpid, sw); // Update event history addSwitchEvent(dpid, EvAction.SWITCH_CONNECTED, "None"); @@ -961,7 +962,7 @@ public class Controller implements IFloodlightProviderService, // TODO: should we wipe the flow table if // alwaysClearFlowsOnSwAdd is set? OTOH this case should // really never happen. - counters.errorSameSwitchReactivated.increment(); + counters.errorSameSwitchReactivated.increment(true); log.error("Switch {} activated but was already active", sw); addSwitchToStore(sw); return; @@ -971,7 +972,7 @@ public class Controller implements IFloodlightProviderService, // This happens either when we have switches with duplicate // DPIDs or when a switch reconnects before we saw the // disconnect - counters.switchWithSameDpidActivated.increment(); + counters.switchWithSameDpidActivated.increment(true); log.warn("New switch added {} for already-added switch {}", sw, oldSw); // We need to disconnect and remove the old switch @@ -1008,7 +1009,7 @@ public class Controller implements IFloodlightProviderService, SwitchUpdateType.ADDED)); addUpdateToQueue(new SwitchUpdate(dpid, SwitchUpdateType.ACTIVATED)); - counters.newSwitchActivated.increment(); + counters.newSwitchActivated.increment(true); } else { // FIXME: switch was in store. check if ports or anything else // has changed and send update. @@ -1027,7 +1028,7 @@ public class Controller implements IFloodlightProviderService, addUpdateToQueue(new SwitchUpdate(dpid, SwitchUpdateType.ACTIVATED)); sendNotificationsIfSwitchDiffers(storedSwitch, sw); - counters.syncedSwitchActivated.increment(); + counters.syncedSwitchActivated.increment(true); if (this.syncedSwitches.isEmpty()) { // we have just activated the last synced switch. I.e., // all previously known switch are now active. Send @@ -1049,11 +1050,11 @@ public class Controller implements IFloodlightProviderService, ImmutablePort port, PortChangeType type) { if (role != Role.MASTER) { - counters.invalidPortsChanged.increment(); + counters.invalidPortsChanged.increment(true); return; } if (!this.activeSwitches.containsKey(sw.getId())) { - counters.invalidPortsChanged.increment(); + counters.invalidPortsChanged.increment(true); return; } // update switch in store @@ -1073,20 +1074,20 @@ public class Controller implements IFloodlightProviderService, */ private synchronized void switchAddedToStore(IOFSwitch sw) { if (role != Role.SLAVE) { - counters.invalidStoreEventWhileMaster.increment(); + counters.invalidStoreEventWhileMaster.increment(true); return; // only read from store if slave } Long dpid = sw.getId(); IOFSwitch oldSw = syncedSwitches.put(dpid, sw); if (oldSw == null) { - counters.newSwitchFromStore.increment(); + counters.newSwitchFromStore.increment(true); addUpdateToQueue(new SwitchUpdate(dpid, SwitchUpdateType.ADDED)); } else { // The switch already exists in storage, see if anything // has changed sendNotificationsIfSwitchDiffers(oldSw, sw); - counters.updatedSwitchFromStore.increment(); + counters.updatedSwitchFromStore.increment(true); } } @@ -1097,19 +1098,19 @@ public class Controller implements IFloodlightProviderService, */ private synchronized void switchRemovedFromStore(long dpid) { if (role != Role.SLAVE) { - counters.invalidStoreEventWhileMaster.increment(); + counters.invalidStoreEventWhileMaster.increment(true); return; // only read from store if slave } IOFSwitch oldSw = syncedSwitches.remove(dpid); if (oldSw != null) { - counters.syncedSwitchRemoved.increment(); + counters.syncedSwitchRemoved.increment(true); addUpdateToQueue(new SwitchUpdate(dpid, SwitchUpdateType.REMOVED)); } else { // TODO: the switch was deleted (tombstone) before we ever // knew about it (or was deleted repeatedly). Can this // happen? When/how? - counters.unknownSwitchRemovedFromStore.increment(); + counters.unknownSwitchRemovedFromStore.increment(true); } } @@ -1124,7 +1125,7 @@ public class Controller implements IFloodlightProviderService, */ public synchronized void switchDisconnected(IOFSwitch sw) { if (role == Role.SLAVE) { - counters.switchDisconnectedWhileSlave.increment(); + counters.switchDisconnectedWhileSlave.increment(true); return; // only react to switch connections when master } long dpid = sw.getId(); @@ -1134,7 +1135,7 @@ public class Controller implements IFloodlightProviderService, // slave as well? addSwitchEvent(dpid, EvAction.SWITCH_DISCONNECTED, "None"); debugEvents.updateEvent(SWITCH_EVENT, new Object[] {dpid, "disconnected"}); - counters.switchDisconnected.increment(); + counters.switchDisconnected.increment(true); IOFSwitch oldSw = this.activeSwitches.get(dpid); if (oldSw != sw) { // This can happen if the disconnected switch was inactive @@ -1173,7 +1174,7 @@ public class Controller implements IFloodlightProviderService, // even throw this error? Should not since all local store // access is synchronized } catch (SyncException e) { - counters.storeSyncError.increment(); + counters.storeSyncError.increment(true); log.error("Could not write switch " + sw.getStringId() + " to sync store:", e); } @@ -1203,7 +1204,7 @@ public class Controller implements IFloodlightProviderService, // even throw this error? Should not since all local store // access is synchronized } catch (SyncException e) { - counters.storeSyncError.increment(); + counters.storeSyncError.increment(true); log.error("Could not write switch " + sw.getStringId() + " to sync store:", e); } @@ -1218,7 +1219,7 @@ public class Controller implements IFloodlightProviderService, try { storeClient.delete(dpid); } catch (SyncException e) { - counters.storeSyncError.increment(); + counters.storeSyncError.increment(true); // ObsoleteVerisonException can't happend because all // store modifications are synchronized log.error("Could not remove switch " + @@ -1256,14 +1257,14 @@ public class Controller implements IFloodlightProviderService, if (role == Role.SLAVE) return; boolean shouldNotifyReadyForReconcile = false; - counters.consolidateStoreRunCount.increment(); + counters.consolidateStoreRunCount.increment(true); log.info("Consolidating synced switches after MASTER transition"); IClosableIterator<Map.Entry<Long,Versioned<SwitchSyncRepresentation>>> iter = null; try { iter = storeClient.entries(); } catch (SyncException e) { - counters.storeSyncError.increment(); + counters.storeSyncError.increment(true); log.error("Failed to read switches from sync store", e); return; } @@ -1279,7 +1280,7 @@ public class Controller implements IFloodlightProviderService, // switch known to the old master that hasn't // reconnected to this controller. counters.switchesNotReconnectingToNewMaster - .increment(); + .increment(true); shouldNotifyReadyForReconcile = true; addUpdateToQueue(new SwitchUpdate(entry.getKey(), SwitchUpdateType.REMOVED)); @@ -1291,7 +1292,7 @@ public class Controller implements IFloodlightProviderService, // stopped reacting to store notifications (due // to MASTER transition) counters.consolidateStoreInconsistencies - .increment(); + .increment(true); } } } @@ -1306,8 +1307,8 @@ public class Controller implements IFloodlightProviderService, // these switches. Iterator<Long> it = this.syncedSwitches.keySet().iterator(); while (it.hasNext()) { - counters.switchesNotReconnectingToNewMaster.increment(); - counters.consolidateStoreInconsistencies.increment(); + counters.switchesNotReconnectingToNewMaster.increment(true); + counters.consolidateStoreInconsistencies.increment(true); Long dpid = it.next(); shouldNotifyReadyForReconcile = true; addUpdateToQueue(new SwitchUpdate(dpid, @@ -1326,7 +1327,7 @@ public class Controller implements IFloodlightProviderService, // that we have written them to the store). for (IOFSwitch sw: this.activeSwitches.values()) { if (addSwitchToStoreIfAbsent(sw)) - counters.consolidateStoreInconsistencies.increment(); + counters.consolidateStoreInconsistencies.increment(true); } } @@ -1404,7 +1405,7 @@ public class Controller implements IFloodlightProviderService, private class ReadyForReconcileUpdate implements IUpdate { @Override public void dispatch() { - counters.readyForReconcile.increment(); + counters.readyForReconcile.increment(true); if (readyForReconcileListeners != null) { for (IReadyForReconcileListener listener: readyForReconcileListeners) { @@ -1480,7 +1481,7 @@ public class Controller implements IFloodlightProviderService, listener.switchRemoved(swId); break; case PORTCHANGED: - counters.switchPortChanged.increment(); + counters.switchPortChanged.increment(true); listener.switchPortChanged(swId, port, changeType); break; case ACTIVATED: @@ -1492,7 +1493,7 @@ public class Controller implements IFloodlightProviderService, // ignore break; case OTHERCHANGE: - counters.switchOtherChange.increment(); + counters.switchOtherChange.increment(true); listener.switchChanged(swId); break; } @@ -1629,7 +1630,6 @@ public class Controller implements IFloodlightProviderService, @Override public void setRole(Role role, String roleChangeDescription) { roleManager.setRole(role, roleChangeDescription); - debugCounters.flushCounters(); } // **************** @@ -1734,11 +1734,11 @@ public class Controller implements IFloodlightProviderService, Ethernet eth = null; if (this.notifiedRole == Role.SLAVE) { - counters.dispatchMessageWhileSlave.increment(); + counters.dispatchMessageWhileSlave.increment(false); // We are SLAVE. Do not dispatch messages to listeners. return; } - counters.dispatchMessage.increment(); + counters.dispatchMessage.increment(false); switch (m.getType()) { case PACKET_IN: @@ -2154,8 +2154,6 @@ public class Controller implements IFloodlightProviderService, } catch (Exception e) { log.error("Exception in controller updates loop", e); } - // Need to flush counters after the notification is dispatched - debugCounters.flushCounters(); } } @@ -2402,7 +2400,7 @@ public class Controller implements IFloodlightProviderService, removedControllerNodeIPs.put(removedControllerID, controllerNodeIPsCache.get(removedControllerID)); controllerNodeIPsCache.clear(); controllerNodeIPsCache.putAll(curControllerNodeIPs); - counters.controllerNodeIpsChanged.increment(); + counters.controllerNodeIpsChanged.increment(true); HAControllerNodeIPUpdate update = new HAControllerNodeIPUpdate( curControllerNodeIPs, addedControllerNodeIPs, removedControllerNodeIPs); diff --git a/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java b/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java index cd4b4af37..40eb3596d 100644 --- a/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java +++ b/src/main/java/net/floodlightcontroller/core/internal/OFChannelHandler.java @@ -199,7 +199,7 @@ class OFChannelHandler if (!requestPending) sendRoleRequest(role); else - counters.roleNotResentBecauseRolePending.increment(); + counters.roleNotResentBecauseRolePending.increment(true); } /** @@ -266,7 +266,7 @@ class OFChannelHandler if (pendingXid == xid && pendingRole == role) { log.debug("Received role reply message from {}, setting role to {}", getSwitchInfoString(), role); - counters.roleReplyReceived.increment(); + counters.roleReplyReceived.increment(true); setSwitchRole(role, RoleRecvStatus.RECEIVED_REPLY); } else { log.debug("Received stale or unexpected role reply from " + @@ -299,7 +299,7 @@ class OFChannelHandler (error.getErrorType() == OFError.OFErrorType. OFPET_BAD_REQUEST.getValue()); if (isBadRequestError) { - counters.roleReplyErrorUnsupported.increment(); + counters.roleReplyErrorUnsupported.increment(true); setSwitchRole(pendingRole, RoleRecvStatus.UNSUPPORTED); } else { // TODO: Is this the right thing to do if we receive @@ -346,7 +346,7 @@ class OFChannelHandler long now = System.currentTimeMillis(); if (now - roleSubmitTime > roleTimeoutMs) { // timeout triggered. - counters.roleReplyTimeout.increment(); + counters.roleReplyTimeout.increment(true); setSwitchRole(pendingRole, RoleRecvStatus.NO_REPLY); } } @@ -712,7 +712,7 @@ class OFChannelHandler // if two controllers are master (even if its only for // a brief period). We might need to see if these errors // persist before we reassert - h.counters.epermErrorWhileSwitchIsMaster.increment(); + h.counters.epermErrorWhileSwitchIsMaster.increment(true); log.warn("Received permission error from switch {} while" + "being master. Reasserting master role.", h.getSwitchInfoString()); @@ -838,7 +838,7 @@ class OFChannelHandler LogMessageDoc.CHECK_CONTROLLER ) void processOFPacketIn(OFChannelHandler h, OFPacketIn m) throws IOException { // we don't expect packetIn while slave, reassert we are slave - h.counters.packetInWhileSwitchIsSlave.increment(); + h.counters.packetInWhileSwitchIsSlave.increment(false); log.warn("Received PacketIn from switch {} while" + "being slave. Reasserting slave role.", h.sw); h.controller.reassertRole(h, Role.SLAVE); @@ -905,7 +905,7 @@ class OFChannelHandler */ protected void unhandledMessageReceived(OFChannelHandler h, OFMessage m) { - h.counters.unhandledMessage.increment(); + h.counters.unhandledMessage.increment(false); if (log.isDebugEnabled()) { String msg = getSwitchStateMessage(h, m, "Ignoring unexpected message"); @@ -1238,13 +1238,12 @@ class OFChannelHandler "specified IP address") public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception { - counters.switchConnected.increment(); + counters.switchConnected.increment(true); channel = e.getChannel(); log.info("New switch connection from {}", channel.getRemoteAddress()); sendHandShakeMessage(OFType.HELLO); setState(ChannelState.WAIT_HELLO); - counters.flushCounters(); } @Override @@ -1257,7 +1256,6 @@ class OFChannelHandler this.sw.setConnected(false); log.info("Disconnected switch {}", getSwitchInfoString()); - counters.flushCounters(); } @Override @@ -1308,31 +1306,31 @@ class OFChannelHandler // switch timeout log.error("Disconnecting switch {} due to read timeout", getSwitchInfoString()); - counters.switchDisconnectReadTimeout.increment(); + counters.switchDisconnectReadTimeout.increment(true); ctx.getChannel().close(); } else if (e.getCause() instanceof HandshakeTimeoutException) { log.error("Disconnecting switch {}: failed to complete handshake", getSwitchInfoString()); - counters.switchDisconnectHandshakeTimeout.increment(); + counters.switchDisconnectHandshakeTimeout.increment(true); ctx.getChannel().close(); } else if (e.getCause() instanceof ClosedChannelException) { log.debug("Channel for sw {} already closed", getSwitchInfoString()); } else if (e.getCause() instanceof IOException) { log.error("Disconnecting switch {} due to IO Error: {}", getSwitchInfoString(), e.getCause().getMessage()); - counters.switchDisconnectIOError.increment(); + counters.switchDisconnectIOError.increment(true); ctx.getChannel().close(); } else if (e.getCause() instanceof SwitchStateException) { log.error("Disconnecting switch {} due to switch state error: {}", getSwitchInfoString(), e.getCause().getMessage()); - counters.switchDisconnectSwitchStateException.increment(); + counters.switchDisconnectSwitchStateException.increment(true); ctx.getChannel().close(); } else if (e.getCause() instanceof MessageParseException) { log.error("Disconnecting switch " + getSwitchInfoString() + " due to message parse failure", e.getCause()); - counters.switchDisconnectParseError.increment(); + counters.switchDisconnectParseError.increment(true); ctx.getChannel().close(); } else if (e.getCause() instanceof StorageException) { log.error("Terminating controller due to storage exception", @@ -1340,14 +1338,13 @@ class OFChannelHandler this.controller.terminate(); } else if (e.getCause() instanceof RejectedExecutionException) { log.warn("Could not process message: queue full"); - counters.rejectedExecutionException.increment(); + counters.rejectedExecutionException.increment(true); } else { log.error("Error while processing message from switch " + getSwitchInfoString(), e.getCause()); - counters.switchDisconnectOtherException.increment(); + counters.switchDisconnectOtherException.increment(true); ctx.getChannel().close(); } - counters.flushCounters(); } @Override @@ -1377,10 +1374,10 @@ class OFChannelHandler } for (OFMessage ofm : msglist) { - counters.messageReceived.increment(); + counters.messageReceived.increment(false); // Per-switch input throttling if (sw != null && sw.inputThrottled(ofm)) { - counters.messageInputThrottled.increment(); + counters.messageInputThrottled.increment(false); continue; } try { diff --git a/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java b/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java index b8ba125f3..f094b3203 100644 --- a/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java +++ b/src/main/java/net/floodlightcontroller/debugcounter/DebugCounter.java @@ -1,6 +1,7 @@ package net.floodlightcontroller.debugcounter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -9,11 +10,14 @@ import java.util.Map; import java.util.Set; import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Sets; + import net.floodlightcontroller.core.module.FloodlightModuleContext; import net.floodlightcontroller.core.module.FloodlightModuleException; import net.floodlightcontroller.core.module.IFloodlightModule; @@ -30,34 +34,22 @@ import net.floodlightcontroller.restserver.IRestApiService; public class DebugCounter implements IFloodlightModule, IDebugCounterService { protected static Logger log = LoggerFactory.getLogger(DebugCounter.class); + /** + * registered counters need a counter id + */ + protected AtomicInteger counterIdCounter = new AtomicInteger(); + /** * The counter value */ protected class MutableLong { long value = 0; public void increment() { value += 1; } + public void increment(long incr) { value += incr; } public long get() { return value; } public void set(long val) { value = val; } } - /** - * Global debug-counter storage across all threads. These are - * updated from the local per thread counters by the flush counters method. - */ - protected ConcurrentHashMap<String, AtomicLong> debugCounters = - new ConcurrentHashMap<String, AtomicLong>(); - - /** - * Thread local debug counters used for maintaining counters local to a thread. - */ - protected final ThreadLocal<Map<String, MutableLong>> threadlocalCounters = - new ThreadLocal<Map<String, MutableLong>>() { - @Override - protected Map<String, MutableLong> initialValue() { - return new HashMap<String, MutableLong>(); - } - }; - /** * protected class to store counter information */ @@ -67,14 +59,20 @@ public class DebugCounter implements IFloodlightModule, IDebugCounterService { CounterType ctype; String moduleName; String counterName; - - public CounterInfo(String name, String desc, CounterType ctype) { - this.moduleCounterName = name; - String[] temp = name.split("-"); - this.moduleName = temp[0]; - this.counterName = temp[1]; + int counterId; + boolean enabled; + Object[] metaData; + + public CounterInfo(int counterId, boolean enabled, Object[] metaData, + String moduleName, String counterName, + String desc, CounterType ctype) { + this.moduleCounterName = moduleName + "/" + counterName; + this.moduleName = moduleName; + this.counterName = counterName; this.counterDesc = desc; this.ctype = ctype; + this.counterId = counterId; + this.enabled = enabled; } public String getModuleCounterName() { return moduleCounterName; } @@ -82,28 +80,103 @@ public class DebugCounter implements IFloodlightModule, IDebugCounterService { public CounterType getCtype() { return ctype; } public String getModuleName() { return moduleName; } public String getCounterName() { return counterName; } + public int getCounterId() { return counterId; } + public boolean isEnabled() { return enabled; } + public Object[] getMetaData() { return metaData; } + } + + //****************** + // Global stores + //****************** + + /** + * Counter info for a debug counter + */ + public class DebugCounterInfo { + CounterInfo cinfo; + AtomicLong cvalue; + + public DebugCounterInfo(CounterInfo cinfo) { + this.cinfo = cinfo; + this.cvalue = new AtomicLong(); + } + public CounterInfo getCounterInfo() { + return cinfo; + } + public Long getCounterValue() { + return cvalue.get(); + } + } + + /** + * Global debug-counter storage across all threads. These are + * updated from the local per thread counters by the flush counters method. + */ + protected static DebugCounterInfo[] allCounters = + new DebugCounterInfo[MAX_COUNTERS]; + + + /** + * per module counters, indexed by the module name and storing three levels + * of Counter information in the form of CounterIndexStore + */ + protected ConcurrentHashMap<String, ConcurrentHashMap<String, CounterIndexStore>> + moduleCounters = new ConcurrentHashMap<String, + ConcurrentHashMap<String, + CounterIndexStore>>(); + + protected class CounterIndexStore { + int index; + Map<String, CounterIndexStore> nextLevel; + + public CounterIndexStore(int index, Map<String,CounterIndexStore> cis) { + this.index = index; + this.nextLevel = cis; + } } /** - * per module counters, indexed by the module name and storing Counter information. + * fast global cache for counter ids that are currently active */ - protected ConcurrentHashMap<String, List<CounterInfo>> moduleCounters = - new ConcurrentHashMap<String, List<CounterInfo>>(); + protected Set<Integer> currentCounters = Collections.newSetFromMap( + new ConcurrentHashMap<Integer,Boolean>()); + + //****************** + // Thread local stores + //****************** /** - * fast global cache for counter names that are currently active + * Thread local storage of counter info */ - Set<String> currentCounters = Collections.newSetFromMap( - new ConcurrentHashMap<String,Boolean>()); + protected class LocalCounterInfo { + boolean enabled; + MutableLong cvalue; + + public LocalCounterInfo(boolean enabled) { + this.enabled = enabled; + this.cvalue = new MutableLong(); + } + } /** - * Thread local cache for counter names that are currently active. + * Thread local debug counters used for maintaining counters local to a thread. */ - protected final ThreadLocal<Set<String>> threadlocalCurrentCounters = - new ThreadLocal<Set<String>>() { + protected final ThreadLocal<LocalCounterInfo[]> threadlocalCounters = + new ThreadLocal<LocalCounterInfo[]>() { @Override - protected Set<String> initialValue() { - return new HashSet<String>(); + protected LocalCounterInfo[] initialValue() { + return new LocalCounterInfo[MAX_COUNTERS]; + } + }; + + /** + * Thread local cache for counter ids that are currently active. + */ + protected final ThreadLocal<Set<Integer>> threadlocalCurrentCounters = + new ThreadLocal<Set<Integer>>() { + @Override + protected Set<Integer> initialValue() { + return new HashSet<Integer>(); } }; @@ -112,122 +185,184 @@ public class DebugCounter implements IFloodlightModule, IDebugCounterService { //******************************* @Override - public boolean registerCounter(String moduleCounterName, String counterDescription, - CounterType counterType) { - if (debugCounters.containsKey(moduleCounterName)) { - log.error("Cannot register counter: {}. Counter already exists", - moduleCounterName); - return false; - } - String[] temp = moduleCounterName.split("-"); - if (temp.length < 2) { - log.error("Cannot register counter: {}. Name not of type " + - " <module name>-<counter name>", moduleCounterName); - return false; - } - - // store counter information on a per module basis - String moduleName = temp[0]; - List<CounterInfo> a; - if (moduleCounters.containsKey(moduleName)) { - a = moduleCounters.get(moduleName); - } else { - a = new ArrayList<CounterInfo>(); - moduleCounters.put(moduleName, a); + public int registerCounter(String moduleName, String counterName, + String counterDescription, CounterType counterType, + Object[] metaData) throws MaxCountersRegistered{ + // check if counter already exists + if (!moduleCounters.containsKey(moduleName)) { + moduleCounters.putIfAbsent(moduleName, + new ConcurrentHashMap<String, CounterIndexStore>()); + } + RetCtrInfo rci = getCounterId(moduleName, counterName); + if (rci.allLevelsFound) { + // counter exists + log.info("Counter exists for {}/{} -- resetting counters", moduleName, + counterName); + resetCounterHierarchy(moduleName, counterName); + return rci.ctrIds[rci.foundUptoLevel-1]; + } + // check for validity of counter + if (rci.levels.length > MAX_HIERARCHY) { + log.error("Registry of counterName {} exceeds max hierachy {}.. aborting", + counterName, MAX_HIERARCHY); + return -1; + } + if (rci.foundUptoLevel < rci.levels.length-1) { + String needToRegister = ""; + for (int i=0; i<=rci.foundUptoLevel; i++) { + needToRegister += rci.levels[i]; + } + log.error("Attempting to register hierarchical counterName {}, "+ + "but parts of hierarchy missing. Please register {} first ", + counterName, needToRegister); + return -1; } - a.add(new CounterInfo(moduleCounterName, counterDescription, counterType)); - // create counter in global map - // and add to counter name cache if it is meant to be always counted - if (counterType != CounterType.COUNT_ON_DEMAND) { - currentCounters.add(moduleCounterName); - debugCounters.put(moduleCounterName, new AtomicLong()); + // get a new counter id + int counterId = counterIdCounter.getAndIncrement(); + if (counterId >= MAX_COUNTERS) { + throw new MaxCountersRegistered(); + } + // create storage for counter + boolean enabled = (counterType == CounterType.ALWAYS_COUNT) ? true : false; + CounterInfo ci = new CounterInfo(counterId, enabled, metaData, moduleName, + counterName, counterDescription, counterType); + allCounters[counterId] = new DebugCounterInfo(ci); + + // account for the new counter in the module counter hierarchy + addToModuleCounterHierarchy(moduleName, counterId, rci); + + // finally add to active counters + if (enabled) { + currentCounters.add(counterId); } - return true; + return counterId; } + @Override - public void updateCounter(String moduleCounterName) { - updateCounter(moduleCounterName, 1); + public void updateCounter(int counterId, boolean flushNow) { + updateCounter(counterId, 1, flushNow); } @Override - public void updateCounter(String moduleCounterName, int incr) { - Map<String, MutableLong> thismap = this.threadlocalCounters.get(); - MutableLong ml = thismap.get(moduleCounterName); - if (ml == null) { - // check locally to see if this counter should be created or not - // FIXME: this is an O(n) operation per update - change to a boolean check - Set<String> thisset = this.threadlocalCurrentCounters.get(); - if (thisset.contains(moduleCounterName)) { - ml = new MutableLong(); - ml.set(ml.get() + incr); - thismap.put(moduleCounterName, ml); + public void updateCounter(int counterId, int incr, boolean flushNow) { + if (counterId < 0 || counterId >= MAX_COUNTERS) return; + + LocalCounterInfo[] thiscounters = this.threadlocalCounters.get(); + if (thiscounters[counterId] == null) { + // seeing this counter for the first time in this thread - create local + // store by consulting global store + DebugCounterInfo dc = allCounters[counterId]; + if (dc != null) { + thiscounters[counterId] = new LocalCounterInfo(dc.cinfo.enabled); + if (dc.cinfo.enabled) { + Set<Integer> thisset = this.threadlocalCurrentCounters.get(); + thisset.add(counterId); + } + } else { + log.error("updateCounter seen locally for counter {} but no global" + + "storage exists for it yet .. not updating", counterId); + return; + } + } + + // update local store if enabled locally for updating + LocalCounterInfo lc = thiscounters[counterId]; + if (lc.enabled) { + lc.cvalue.increment(incr); + if (flushNow) { + DebugCounterInfo dc = allCounters[counterId]; + if (dc.cinfo.enabled) { + // globally enabled - flush now + dc.cvalue.addAndGet(lc.cvalue.get()); + lc.cvalue.set(0); + } else { + // global counter is disabled - don't flush, disable locally + lc.enabled = false; + Set<Integer> thisset = this.threadlocalCurrentCounters.get(); + thisset.remove(counterId); + } } - } else { - ml.increment(); } + } @Override public void flushCounters() { - Map<String, MutableLong> thismap = this.threadlocalCounters.get(); - ArrayList<String> deleteKeys = new ArrayList<String>(); - for (String key : thismap.keySet()) { - MutableLong curval = thismap.get(key); - long delta = curval.get(); - if (delta > 0) { - AtomicLong ctr = debugCounters.get(key); - if (ctr == null) { - // The global counter does not exist possibly because it has been - // disabled. It should thus be removed from the thread-local - // map (the counter) and set (the counter name). Removing it - // from the threadlocal set ensures that the counter will not be - // recreated (see updateCounter) - Set<String> thisset = this.threadlocalCurrentCounters.get(); - thisset.remove(key); - deleteKeys.add(key); + LocalCounterInfo[] thiscounters = this.threadlocalCounters.get(); + Set<Integer> thisset = this.threadlocalCurrentCounters.get(); + ArrayList<Integer> temp = new ArrayList<Integer>(); + + for (int counterId : thisset) { + LocalCounterInfo lc = thiscounters[counterId]; + if (lc.cvalue.get() > 0) { + DebugCounterInfo dc = allCounters[counterId]; + if (dc.cinfo.enabled) { + // globally enabled - flush now + dc.cvalue.addAndGet(lc.cvalue.get()); + lc.cvalue.set(0); } else { - ctr.addAndGet(delta); - curval.set(0); + // global counter is disabled - don't flush, disable locally + lc.enabled = false; + temp.add(counterId); } } } - for (String dkey : deleteKeys) - thismap.remove(dkey); + for (int cId : temp) { + thisset.remove(cId); + } - // At this point it is also possible that the threadlocal map/set does not - // include a counter that has been enabled and is present in the global - // currentCounters set. If so we need to sync such state so that the - // thread local counter can be created (in the updateCounter method) - Set<String> thisset = this.threadlocalCurrentCounters.get(); - if (thisset.size() != currentCounters.size()) { - thisset.addAll(currentCounters); + // At this point it is possible that the thread-local set does not + // include a counter that has been enabled and is present in the global set. + // We need to sync thread-local currently enabled set of counterIds with + // the global set. + Sets.SetView<Integer> sv = Sets.difference(currentCounters, thisset); + for (int counterId : sv) { + if (thiscounters[counterId] != null) thiscounters[counterId].enabled = true; + thisset.add(counterId); } } @Override - public void resetCounter(String moduleCounterName) { - if (debugCounters.containsKey(moduleCounterName)) { - debugCounters.get(moduleCounterName).set(0); + public void resetCounterHierarchy(String moduleName, String counterName) { + RetCtrInfo rci = getCounterId(moduleName, counterName); + if (!rci.allLevelsFound) { + String missing = rci.levels[rci.foundUptoLevel]; + log.error("Cannot reset counter hierarchy - missing counter {}", missing); + return; + } + // reset at this level + allCounters[rci.ctrIds[rci.foundUptoLevel-1]].cvalue.set(0); + // reset all levels below + ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci); + for (int index : resetIds) { + allCounters[index].cvalue.set(0); } } @Override public void resetAllCounters() { - for (AtomicLong v : debugCounters.values()) { - v.set(0); + RetCtrInfo rci = new RetCtrInfo(); + rci.levels = "".split("/"); + for (String moduleName : moduleCounters.keySet()) { + ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci); + for (int index : resetIds) { + allCounters[index].cvalue.set(0); + } } } @Override public void resetAllModuleCounters(String moduleName) { - List<CounterInfo> cil = moduleCounters.get(moduleName); - if (cil != null) { - for (CounterInfo ci : cil) { - if (debugCounters.containsKey(ci.moduleCounterName)) { - debugCounters.get(ci.moduleCounterName).set(0); - } + Map<String, CounterIndexStore> target = moduleCounters.get(moduleName); + RetCtrInfo rci = new RetCtrInfo(); + rci.levels = "".split("/"); + + if (target != null) { + ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci); + for (int index : resetIds) { + allCounters[index].cvalue.set(0); } } else { if (log.isDebugEnabled()) @@ -236,65 +371,66 @@ public class DebugCounter implements IFloodlightModule, IDebugCounterService { } @Override - public void enableCtrOnDemand(String moduleCounterName) { - currentCounters.add(moduleCounterName); - debugCounters.putIfAbsent(moduleCounterName, new AtomicLong()); + public void enableCtrOnDemand(String moduleName, String counterName) { + RetCtrInfo rci = getCounterId(moduleName, counterName); + if (!rci.allLevelsFound) { + String missing = rci.levels[rci.foundUptoLevel]; + log.error("Cannot enable counter - counter not found {}", missing); + return; + } + // enable specific counter + DebugCounterInfo dc = allCounters[rci.ctrIds[rci.foundUptoLevel-1]]; + dc.cinfo.enabled = true; + currentCounters.add(dc.cinfo.counterId); } @Override - public void disableCtrOnDemand(String moduleCounterName) { - String[] temp = moduleCounterName.split("-"); - if (temp.length < 2) { - log.error("moduleCounterName {} not recognized", moduleCounterName); + public void disableCtrOnDemand(String moduleName, String counterName) { + RetCtrInfo rci = getCounterId(moduleName, counterName); + if (!rci.allLevelsFound) { + String missing = rci.levels[rci.foundUptoLevel]; + log.error("Cannot disable counter - counter not found {}", missing); return; } - String moduleName = temp[0]; - List<CounterInfo> cil = moduleCounters.get(moduleName); - for (CounterInfo ci : cil) { - if (ci.moduleCounterName.equals(moduleCounterName) && - ci.ctype == CounterType.COUNT_ON_DEMAND) { - currentCounters.remove(moduleCounterName); - debugCounters.remove(moduleCounterName); - return; - } + // disable specific counter + DebugCounterInfo dc = allCounters[rci.ctrIds[rci.foundUptoLevel-1]]; + if (dc.cinfo.ctype == CounterType.COUNT_ON_DEMAND) { + dc.cinfo.enabled = false; + dc.cvalue.set(0); + currentCounters.remove(dc.cinfo.counterId); } } @Override - public DebugCounterInfo getCounterValue(String moduleCounterName) { - if (!debugCounters.containsKey(moduleCounterName)) return null; - long counterValue = debugCounters.get(moduleCounterName).longValue(); - - String[] temp = moduleCounterName.split("-"); - if (temp.length < 2) { - log.error("moduleCounterName {} not recognized", moduleCounterName); + public List<DebugCounterInfo> getCounterHierarchy(String moduleName, + String counterName) { + RetCtrInfo rci = getCounterId(moduleName, counterName); + if (!rci.allLevelsFound) { + String missing = rci.levels[rci.foundUptoLevel]; + log.error("Cannot fetch counter - counter not found {}", missing); return null; } - String moduleName = temp[0]; - List<CounterInfo> cil = moduleCounters.get(moduleName); - for (CounterInfo ci : cil) { - if (ci.moduleCounterName.equals(moduleCounterName)) { - DebugCounterInfo dci = new DebugCounterInfo(); - dci.counterInfo = ci; - dci.counterValue = counterValue; - return dci; - } + ArrayList<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>(); + // get counter and all below it + DebugCounterInfo dc = allCounters[rci.ctrIds[rci.foundUptoLevel-1]]; + dcilist.add(dc); + ArrayList<Integer> belowIds = getHierarchyBelow(moduleName, rci); + for (int index : belowIds) { + dcilist.add(allCounters[index]); } - return null; + return dcilist; } @Override public List<DebugCounterInfo> getAllCounterValues() { List<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>(); - for (List<CounterInfo> cil : moduleCounters.values()) { - for (CounterInfo ci : cil) { - AtomicLong ctr = debugCounters.get(ci.moduleCounterName); - if (ctr != null) { - DebugCounterInfo dci = new DebugCounterInfo(); - dci.counterInfo = ci; - dci.counterValue = ctr.longValue(); - dcilist.add(dci); - } + RetCtrInfo rci = new RetCtrInfo(); + rci.levels = "".split("/"); + + for (String moduleName : moduleCounters.keySet()) { + ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci); + for (int index : resetIds) { + dcilist.add(allCounters[index]); } } return dcilist; @@ -303,36 +439,27 @@ public class DebugCounter implements IFloodlightModule, IDebugCounterService { @Override public List<DebugCounterInfo> getModuleCounterValues(String moduleName) { List<DebugCounterInfo> dcilist = new ArrayList<DebugCounterInfo>(); + RetCtrInfo rci = new RetCtrInfo(); + rci.levels = "".split("/"); + if (moduleCounters.containsKey(moduleName)) { - List<CounterInfo> cil = moduleCounters.get(moduleName); - for (CounterInfo ci : cil) { - AtomicLong ctr = debugCounters.get(ci.moduleCounterName); - if (ctr != null) { - DebugCounterInfo dci = new DebugCounterInfo(); - dci.counterInfo = ci; - dci.counterValue = ctr.longValue(); - dcilist.add(dci); - } + ArrayList<Integer> resetIds = getHierarchyBelow(moduleName, rci); + for (int index : resetIds) { + dcilist.add(allCounters[index]); } } return dcilist; } @Override - public boolean containsMCName(String moduleCounterName) { - if (debugCounters.containsKey(moduleCounterName)) return true; - // it is possible that the counter may be disabled - for (List<CounterInfo> cil : moduleCounters.values()) { - for (CounterInfo ci : cil) { - if (ci.moduleCounterName.equals(moduleCounterName)) - return true; - } - } - return false; + public boolean containsModuleCounterName(String moduleName, String counterName) { + if (!moduleCounters.containsKey(moduleName)) return false; + RetCtrInfo rci = getCounterId(moduleName, counterName); + return rci.allLevelsFound; } @Override - public boolean containsModName(String moduleName) { + public boolean containsModuleName(String moduleName) { return (moduleCounters.containsKey(moduleName)) ? true : false; } @@ -340,12 +467,155 @@ public class DebugCounter implements IFloodlightModule, IDebugCounterService { // Internal Methods //******************************* + protected class RetCtrInfo { + boolean allLevelsFound; // counter indices found all the way down the hierarchy + boolean hierarchical; // true if counterName is hierarchical + int foundUptoLevel; + int[] ctrIds; + String[] levels; + + public RetCtrInfo() { + ctrIds = new int[MAX_HIERARCHY]; + for (int i=0; i<MAX_HIERARCHY; i++) { + ctrIds[i] = -1; + } + } + + @Override + public boolean equals(Object oth) { + if (!(oth instanceof RetCtrInfo)) return false; + RetCtrInfo other = (RetCtrInfo)oth; + if (other.allLevelsFound != this.allLevelsFound) return false; + if (other.hierarchical != this.hierarchical) return false; + if (other.foundUptoLevel != this.foundUptoLevel) return false; + if (!Arrays.equals(other.ctrIds, this.ctrIds)) return false; + if (!Arrays.equals(other.levels, this.levels)) return false; + return true; + } + + } + + protected RetCtrInfo getCounterId(String moduleName, String counterName) { + RetCtrInfo rci = new RetCtrInfo(); + Map<String, CounterIndexStore> templevel = moduleCounters.get(moduleName); + rci.levels = counterName.split("/"); + if (rci.levels.length > 1) rci.hierarchical = true; + if (templevel == null) return rci; + + /* + if (rci.levels.length > MAX_HIERARCHY) { + // chop off all array elems greater that MAX_HIERARCHY + String[] temp = new String[MAX_HIERARCHY]; + System.arraycopy(rci.levels, 0, temp, 0, MAX_HIERARCHY); + rci.levels = temp; + } + */ + for (int i=0; i<rci.levels.length; i++) { + if (templevel != null) { + CounterIndexStore cis = templevel.get(rci.levels[i]) ; + if (cis == null) { + // could not find counterName part at this level + break; + } else { + rci.ctrIds[i] = cis.index; + templevel = cis.nextLevel; + rci.foundUptoLevel++; + if (i == rci.levels.length-1) { + rci.allLevelsFound = true; + } + } + } else { + // there are no more levels, which means that some part of the + // counterName has no corresponding map + break; + } + } + return rci; + } + + protected void addToModuleCounterHierarchy(String moduleName, int counterId, + RetCtrInfo rci) { + Map<String, CounterIndexStore> target = moduleCounters.get(moduleName); + if (target == null) return; + CounterIndexStore cis = null; + + for (int i=0; i<rci.foundUptoLevel; i++) { + cis = target.get(rci.levels[i]); + target = cis.nextLevel; + } + if (cis != null) { + if (cis.nextLevel == null) + cis.nextLevel = new ConcurrentHashMap<String, CounterIndexStore>(); + cis.nextLevel.put(rci.levels[rci.foundUptoLevel], + new CounterIndexStore(counterId, null)); + } else { + target.put(rci.levels[rci.foundUptoLevel], + new CounterIndexStore(counterId, null)); + } + } + + // given a partial hierarchical counter, return the rest of the hierarchy + protected ArrayList<Integer> getHierarchyBelow(String moduleName, RetCtrInfo rci) { + Map<String, CounterIndexStore> target = moduleCounters.get(moduleName); + CounterIndexStore cis = null; + ArrayList<Integer> retval = new ArrayList<Integer>(); + if (target == null) return retval; + + // get to the level given + for (int i=0; i<rci.foundUptoLevel; i++) { + cis = target.get(rci.levels[i]); + target = cis.nextLevel; + } + + if (target == null || rci.foundUptoLevel == MAX_HIERARCHY) { + // no more levels + return retval; + } else { + // recursively get all ids + getIdsAtLevel(target, retval, rci.foundUptoLevel+1); + } + + return retval; + } + + protected void getIdsAtLevel(Map<String, CounterIndexStore> hcy, + ArrayList<Integer> retval, int level) { + if (level > MAX_HIERARCHY) return; + if (hcy == null || retval == null) return; + + // Can return the counter names as well but for now ids are enough. + for (CounterIndexStore cistemp : hcy.values()) { + retval.add(cistemp.index); // value at this level + if (cistemp.nextLevel != null) { + getIdsAtLevel(cistemp.nextLevel, retval, level+1); + } + } + } + protected void printAllCounters() { - for (List<CounterInfo> cilist : moduleCounters.values()) { - for (CounterInfo ci : cilist) { - log.info("Countername {} Countervalue {}", new Object[] { - ci.moduleCounterName, debugCounters.get(ci.moduleCounterName) - }); + Set<String> keys = moduleCounters.keySet(); + for (String key : keys) { + log.info("ModuleName: {}", key); + Map<String, CounterIndexStore> lev1 = moduleCounters.get(key); + for (String key1 : lev1.keySet()) { + CounterIndexStore cis1 = lev1.get(key1); + log.info(" L1 {}:{}", key1, new Object[] {cis1.index, cis1.nextLevel}); + if (cis1.nextLevel != null) { + Map<String, CounterIndexStore> lev2 = cis1.nextLevel; + for (String key2 : lev2.keySet()) { + CounterIndexStore cis2 = lev2.get(key2); + log.info(" L2 {}:{}", key2, new Object[] {cis2.index, + cis2.nextLevel}); + if (cis2.nextLevel != null) { + Map<String, CounterIndexStore> lev3 = cis2.nextLevel; + for (String key3 : lev3.keySet()) { + CounterIndexStore cis3 = lev3.get(key3); + log.info(" L3 {}:{}", key3, new Object[] {cis3.index, + cis3.nextLevel}); + } + } + } + } } } } diff --git a/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java b/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java index d1d255309..2460086ed 100644 --- a/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java +++ b/src/main/java/net/floodlightcontroller/debugcounter/IDebugCounterService.java @@ -1,7 +1,8 @@ package net.floodlightcontroller.debugcounter; import net.floodlightcontroller.core.module.IFloodlightService; -import net.floodlightcontroller.debugcounter.DebugCounter.CounterInfo; +import net.floodlightcontroller.debugcounter.DebugCounter.DebugCounterInfo; + import java.util.List; public interface IDebugCounterService extends IFloodlightService { @@ -12,23 +13,26 @@ public interface IDebugCounterService extends IFloodlightService { */ public enum CounterType { ALWAYS_COUNT, - COUNT_ON_DEMAND, - WARN, - ERROR + COUNT_ON_DEMAND } - public class DebugCounterInfo { - CounterInfo counterInfo; - Long counterValue; + /** + * A limit on the maximum number of counters that can be created + */ + public static final int MAX_COUNTERS = 10000; - public CounterInfo getCounterInfo() { - return counterInfo; - } - public Long getCounterValue() { - return counterValue; - } + /** + * exception thrown when MAX_COUNTERS have been registered + */ + public class MaxCountersRegistered extends Exception { + private static final long serialVersionUID = 3173747663719376745L; } + /** + * maximum level of hierarchical counters + */ + public static final int MAX_HIERARCHY = 3; + /** * All modules that wish to have the DebugCounterService count for them, must * register their counters by making this call (typically from that module's @@ -50,21 +54,22 @@ public interface IDebugCounterService extends IFloodlightService { * @return false if the counter has already been registered * or if the moduleCounterName is not as expected. */ - public boolean registerCounter(String moduleCounterName, String counterDescription, - CounterType counterType); + public int registerCounter(String moduleName, String counterName, + String counterDescription, CounterType counterType, + Object[] metaData) throws MaxCountersRegistered; /** * Increments the counter by 1, if the counter is meant to be always counted, * or if the counter has been enabled for counting. * @param moduleCounterName the registered counter name. */ - public void updateCounter(String moduleCounterName); + void updateCounter(int counterId, boolean flushNow); /** * Increments the counter by the number specified * @param moduleCounterName the registered counter name. */ - public void updateCounter(String moduleCounterName, int incr); + void updateCounter(int counterId, int incr, boolean flushNow); /** * Update the global counter map with values from the thread local maps. This @@ -80,7 +85,7 @@ public interface IDebugCounterService extends IFloodlightService { * zero with a get call as it may get updated between the reset and get calls. * @param moduleCounterName the registered counter name. */ - public void resetCounter(String moduleCounterName); + void resetCounterHierarchy(String moduleName, String counterName); /** * Resets the values of all counters that are currently enabled to zero. @@ -106,7 +111,7 @@ public interface IDebugCounterService extends IFloodlightService { * * @param moduleCounterName the registered counter name. */ - public void enableCtrOnDemand(String moduleCounterName); + public void enableCtrOnDemand(String moduleName, String counterName); /** * This method applies only to CounterType.ALWAYS_COUNT. It is used to disable @@ -115,7 +120,7 @@ public interface IDebugCounterService extends IFloodlightService { * * @param moduleCounterName the registered counter name. */ - public void disableCtrOnDemand(String moduleCounterName); + public void disableCtrOnDemand(String moduleName, String counterName); /** * Get counter value and associated information for a specific counter if it @@ -124,7 +129,8 @@ public interface IDebugCounterService extends IFloodlightService { * @param moduleCounterName * @return DebugCounterInfo or null if the counter could not be found */ - public DebugCounterInfo getCounterValue(String moduleCounterName); + public List<DebugCounterInfo> getCounterHierarchy(String moduleName, + String counterName); /** * Get counter values and associated information for all active counters @@ -151,7 +157,7 @@ public interface IDebugCounterService extends IFloodlightService { * @param param * @return false if moduleCounterName is not a registered counter */ - public boolean containsMCName(String moduleCounterName); + public boolean containsModuleCounterName(String moduleName, String counterName); /** * Convenience method to figure out if the the given 'moduleName' corresponds @@ -162,6 +168,7 @@ public interface IDebugCounterService extends IFloodlightService { * @param param * @return false if moduleName is not a registered counter */ - public boolean containsModName(String moduleName); + public boolean containsModuleName(String moduleName); + } diff --git a/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java b/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java index f11a0dfd9..d72a028af 100644 --- a/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java +++ b/src/main/java/net/floodlightcontroller/debugcounter/NullDebugCounter.java @@ -10,15 +10,10 @@ import net.floodlightcontroller.core.module.FloodlightModuleContext; import net.floodlightcontroller.core.module.FloodlightModuleException; import net.floodlightcontroller.core.module.IFloodlightModule; import net.floodlightcontroller.core.module.IFloodlightService; +import net.floodlightcontroller.debugcounter.DebugCounter.DebugCounterInfo; public class NullDebugCounter implements IFloodlightModule, IDebugCounterService { - @Override - public boolean registerCounter(String moduleCounterName, - String counterDescription, - CounterType counterType) { - return false; - } @Override public Collection<Class<? extends IFloodlightService>> @@ -57,48 +52,41 @@ public class NullDebugCounter implements IFloodlightModule, IDebugCounterService } - @Override - public void updateCounter(String moduleCounterName) { - - } - @Override - public void updateCounter(String moduleCounterName, int incr) { - - } - @Override public void flushCounters() { } @Override - public void resetCounter(String moduleCounterName) { + public void resetAllCounters() { } @Override - public void resetAllCounters() { + public void resetAllModuleCounters(String moduleName) { } + @Override - public void resetAllModuleCounters(String moduleName) { + public void resetCounterHierarchy(String moduleName, String counterName) { } @Override - public void enableCtrOnDemand(String moduleCounterName) { + public void enableCtrOnDemand(String moduleName, String counterName) { } @Override - public void disableCtrOnDemand(String moduleCounterName) { + public void disableCtrOnDemand(String moduleName, String counterName) { } @Override - public DebugCounterInfo getCounterValue(String moduleCounterName) { + public List<DebugCounterInfo> getCounterHierarchy(String moduleName, + String counterName) { return null; } @@ -113,13 +101,38 @@ public class NullDebugCounter implements IFloodlightModule, IDebugCounterService } @Override - public boolean containsMCName(String moduleCounterName) { + public boolean containsModuleCounterName(String moduleName, + String counterName) { return false; } @Override - public boolean containsModName(String moduleName) { + public boolean containsModuleName(String moduleName) { return false; } + @Override + public + int + registerCounter(String moduleName, String counterName, + String counterDescription, + CounterType counterType, Object[] metaData) + throws MaxCountersRegistered { + // TODO Auto-generated method stub + return 0; + } + + @Override + public void updateCounter(int counterId, boolean flushNow) { + // TODO Auto-generated method stub + + } + + @Override + public void updateCounter(int counterId, int incr, boolean flushNow) { + // TODO Auto-generated method stub + + } + + } diff --git a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java index 1e590198a..6773b3d79 100644 --- a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java +++ b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterResource.java @@ -4,23 +4,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import net.floodlightcontroller.debugcounter.IDebugCounterService.DebugCounterInfo; import org.restlet.resource.Get; import org.restlet.resource.Post; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.floodlightcontroller.debugcounter.DebugCounter.DebugCounterInfo; + /** - * Return the debug counter data for the get rest-api call - * - * URI must be in one of the following forms: - * "http://{controller-hostname}:8080/wm/counters/{param}" - * - * where {param} must be one of (no quotes): - * "all" returns value/info on all active counters. - * "{moduleName}" returns value/info on all active counters for the specified module. - * "{moduleCounterName}" returns value/info for specific counter if it is active. + * Web interface for Debug Counters * * @author Saurav */ @@ -50,7 +43,8 @@ public class DebugCounterResource extends DebugCounterResourceBase { } public enum Option { - ALL, ONE_MODULE, ONE_MODULE_COUNTER, ERROR_BAD_MODULE_NAME, ERROR_BAD_PARAM, + ALL, ONE_MODULE, MODULE_COUNTER_HIERARCHY, ERROR_BAD_MODULE_NAME, + ERROR_BAD_PARAM, ERROR_BAD_MODULE_COUNTER_NAME } @@ -71,7 +65,7 @@ public class DebugCounterResource extends DebugCounterResourceBase { this.enable = enable; } } - + public static class ResetOutput { String error = null; @@ -82,114 +76,193 @@ public class DebugCounterResource extends DebugCounterResourceBase { this.error = error; } } - + + /** + * Reset or enable/disable counters + * + * If using curl: + * curl -X POST -d DATA -H "Content-Type: application/json" URL + * where DATA must be one of the following: + * {\"reset\":true} to reset counters + * {\"enable\":true} to enable counter + * {\"enable\":false} to disable counter + * and URL must be in one of the following forms: + * "http://{controller-hostname}:8080/wm/debugcounter/{param1}/{param2}/{param3}/{param4} + * + * {param1} can be null, 'all' or the name of a module {moduleName}. + * {param2}/{param3}/{param4} refer to hierarchical counter names. + * + * The Reset command will reset the counter specified as well as all counters + * in the hierarchical levels below. For example, if a counter hierarchy exists + * as switch/00:00:00:00:01:02:03:04/pktin/drops, then a reset command with just + * the moduleName (param1=switch) and counterName (param2=00:00:00:00:01:02:03:04) + * will reset all counters for that switch. Continuing the example - + * for a counterName (param2=00:00:00:00:01:02:03:04 and param3=pktin), the reset + * command will remove all pktin counters for that switch. + * + * The enable/disable command will ONLY disable a specific counter (and only if + * that counter is of CounterType.ON_DEMAND) + * It will not enable/disable counters at any other hierarchical level. + * + */ @Post public ResetOutput postHandler(CounterPost postData) { ResetOutput output = new ResetOutput(); - - String param = (String)getRequestAttributes().get("param"); - if (postData.getReset() != null && postData.getReset()) { - Option choice = Option.ERROR_BAD_PARAM; - - if (param == null) { - param = "all"; - choice = Option.ALL; - } else if (param.equals("all")) { - choice = Option.ALL; - } else if (param.contains("-")) { - // differentiate between disabled and non-existing counters - boolean isRegistered = debugCounter.containsMCName(param); - if (isRegistered) { - choice = Option.ONE_MODULE_COUNTER; - } else { - choice = Option.ERROR_BAD_MODULE_COUNTER_NAME; - } - } else { - boolean isRegistered = debugCounter.containsModName(param); - if (isRegistered) { - choice = Option.ONE_MODULE; - } else { - choice = Option.ERROR_BAD_MODULE_NAME; + Option choice = Option.ERROR_BAD_PARAM; + String param1 = (String)getRequestAttributes().get("param1"); + String param2 = (String)getRequestAttributes().get("param2"); + String param3 = (String)getRequestAttributes().get("param3"); + String param4 = (String)getRequestAttributes().get("param4"); + String moduleName = ""; + + if (param1 == null) { + moduleName = "all"; + choice = Option.ALL; + } else if (param1.equals("all")) { + moduleName = "all"; + choice = Option.ALL; + } + + String counterName = ""; + if (param2 != null) { + counterName += "/" + param2; + if (param3 != null) { + counterName += "/" + param3; + if (param4 != null) { + counterName += "/" + param4; } } + } - switch (choice) { - case ALL: - debugCounter.resetAllCounters(); - break; - case ONE_MODULE: - debugCounter.resetAllModuleCounters(param); - break; - case ONE_MODULE_COUNTER: - debugCounter.resetCounter(param); - break; - case ERROR_BAD_MODULE_NAME: - output.error = "Module name has no corresponding registered counters"; - break; - case ERROR_BAD_MODULE_COUNTER_NAME: - output.error = "Counter not registered"; - break; - case ERROR_BAD_PARAM: - output.error = "Bad param"; + if (!moduleName.equals("all") && counterName.equals("")) { + // only module name specified + boolean isRegistered = debugCounter.containsModuleName(param1); + if (isRegistered) { + choice = Option.ONE_MODULE; + } else { + choice = Option.ERROR_BAD_MODULE_NAME; + } + } else if (!moduleName.equals("all") && !counterName.equals("")) { + // both module and counter names specified + boolean isRegistered = debugCounter.containsModuleCounterName(moduleName, + counterName); + if (isRegistered) { + choice = Option.MODULE_COUNTER_HIERARCHY; + } else { + choice = Option.ERROR_BAD_MODULE_COUNTER_NAME; } } - if (output.getError() != null) return output; - + boolean reset = false; + boolean turnOnOff = false; + if (postData.getReset() != null && postData.getReset()) { + reset = true; + } if (postData.getEnable() != null) { - if (!param.contains("-")) { - output.error = "Specified moduleCounterName is not of type " + - "<moduleName>-<counterName>."; - return output; - } + turnOnOff = true; + } - if (postData.getEnable()) { - debugCounter.enableCtrOnDemand(param); - } else { - debugCounter.disableCtrOnDemand(param); - } + switch (choice) { + case ALL: + if (reset) debugCounter.resetAllCounters(); + break; + case ONE_MODULE: + if (reset) debugCounter.resetAllModuleCounters(moduleName); + break; + case MODULE_COUNTER_HIERARCHY: + if (reset) + debugCounter.resetCounterHierarchy(moduleName, counterName); + else if (turnOnOff && postData.getEnable()) + debugCounter.enableCtrOnDemand(moduleName, counterName); + else if (turnOnOff && !postData.getEnable()) + debugCounter.disableCtrOnDemand(moduleName, counterName); + break; + case ERROR_BAD_MODULE_NAME: + output.error = "Module name has no corresponding registered counters"; + break; + case ERROR_BAD_MODULE_COUNTER_NAME: + output.error = "Counter not registered"; + break; + case ERROR_BAD_PARAM: + output.error = "Bad param"; } return output; } + /** + * Return the debug counter data for the get rest-api call + * + * URI must be in one of the following forms: + * "http://{controller-hostname}:8080/wm/debugcounter/{param1}/{param2}/{param3}/{param4}" + * + * where {param1} must be one of (no quotes): + * null if nothing is given then by default all + * counters are returned + * "all" returns value/info on all counters. + * "{moduleName}" returns value/info on all counters for + * the specified module 'moduelName'. + * & {param2}/{param3}/{param4} refer to hierarchical counter names. + * eg. 00:00:00:00:01:02:03:04/pktin/drops + * where leaving out any of the params returns + * all counters in the hierarchical level below. + * So giving just the switch dpid will fetch + * all counters for that switch. + */ @Get public DebugCounterInfoOutput handleCounterInfoQuery() { DebugCounterInfoOutput output = new DebugCounterInfoOutput(); Option choice = Option.ERROR_BAD_PARAM; - String param = (String)getRequestAttributes().get("param"); - if (param == null) { - param = "all"; + String param1 = (String)getRequestAttributes().get("param1"); + String param2 = (String)getRequestAttributes().get("param2"); + String param3 = (String)getRequestAttributes().get("param3"); + String param4 = (String)getRequestAttributes().get("param4"); + + if (param1 == null) { + param1 = "all"; choice = Option.ALL; - } else if (param.equals("all")) { + } else if (param1.equals("all")) { choice = Option.ALL; - } else if (param.contains("-")) { - // differentiate between disabled and non-existing counters - boolean isRegistered = debugCounter.containsMCName(param); + } + + String counterName = ""; + if (param2 != null) { + counterName += "/" + param2; + if (param3 != null) { + counterName += "/" + param3; + if (param4 != null) { + counterName += "/" + param4; + } + } + boolean isRegistered = debugCounter.containsModuleCounterName(param1, counterName); if (isRegistered) { - choice = Option.ONE_MODULE_COUNTER; + choice = Option.MODULE_COUNTER_HIERARCHY; } else { choice = Option.ERROR_BAD_MODULE_COUNTER_NAME; } } else { - boolean isRegistered = debugCounter.containsModName(param); - if (isRegistered) { - choice = Option.ONE_MODULE; - } else { - choice = Option.ERROR_BAD_MODULE_NAME; + if (!param1.equals("all")) { + // get all counters for a single module + boolean isRegistered = debugCounter.containsModuleName(param1); + if (isRegistered) { + choice = Option.ONE_MODULE; + } else { + choice = Option.ERROR_BAD_MODULE_NAME; + } } } switch (choice) { case ALL: - poplulateAllCounters(debugCounter.getAllCounterValues(), output); + populateCounters(debugCounter.getAllCounterValues(), output); break; case ONE_MODULE: - populateModuleCounters(debugCounter.getModuleCounterValues(param), output); + populateCounters(debugCounter.getModuleCounterValues(param1), output); break; - case ONE_MODULE_COUNTER: - populateSingleCounter(debugCounter.getCounterValue(param), output); + case MODULE_COUNTER_HIERARCHY: + populateCounters(debugCounter.getCounterHierarchy(param1, counterName), + output); break; case ERROR_BAD_MODULE_NAME: output.error = "Module name has no corresponding registered counters"; @@ -212,16 +285,9 @@ public class DebugCounterResource extends DebugCounterResourceBase { debugCounterInfo); } - private void populateModuleCounters(List<DebugCounterInfo> moduleCounterValues, + private void populateCounters(List<DebugCounterInfo> counterValues, DebugCounterInfoOutput output) { - for (DebugCounterInfo dci : moduleCounterValues) { - populateSingleCounter(dci, output); - } - } - - private void poplulateAllCounters(List<DebugCounterInfo> allCounterValues, - DebugCounterInfoOutput output) { - for (DebugCounterInfo dci : allCounterValues) { + for (DebugCounterInfo dci : counterValues) { populateSingleCounter(dci, output); } } diff --git a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java index d0d46b578..8b7b2fa6f 100644 --- a/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java +++ b/src/main/java/net/floodlightcontroller/debugcounter/web/DebugCounterRoutable.java @@ -10,13 +10,21 @@ public class DebugCounterRoutable implements RestletRoutable { @Override public String basePath() { - return "/wm/counters"; + return "/wm/debugcounter"; } @Override public Restlet getRestlet(Context context) { Router router = new Router(context); - router.attach("/{param}", DebugCounterResource.class); + router.attach("/{param1}/{param2}/{param3}/{param4}/", DebugCounterResource.class); + router.attach("/{param1}/{param2}/{param3}/{param4}", DebugCounterResource.class); + router.attach("/{param1}/{param2}/{param3}/", DebugCounterResource.class); + router.attach("/{param1}/{param2}/{param3}", DebugCounterResource.class); + router.attach("/{param1}/{param2}/", DebugCounterResource.class); + router.attach("/{param1}/{param2}", DebugCounterResource.class); + router.attach("/{param1}/", DebugCounterResource.class); + router.attach("/{param1}", DebugCounterResource.class); + router.attach("/", DebugCounterResource.class); return router; } } diff --git a/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java b/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java index 0201bd3c1..d5c0719e2 100644 --- a/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java +++ b/src/main/java/net/floodlightcontroller/debugevent/DebugEvent.java @@ -208,7 +208,7 @@ public class DebugEvent implements IFloodlightModule, IDebugEventService { et, formatStr, eventDescription, eventName, moduleName, flushImmediately); allEvents[eventId] = new DebugEventHistory(ei, bufferCapacity); - if (enabled && eventId < MAX_EVENTS) { + if (enabled) { currentEvents.add(eventId); } diff --git a/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java b/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java index c98d3dcb7..9855c4780 100644 --- a/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java +++ b/src/main/java/net/floodlightcontroller/debugevent/IDebugEventService.java @@ -18,7 +18,7 @@ public interface IDebugEventService extends IFloodlightService { } /** - * A limit on the maximum number of event types that can be created + * A limit on the maximum number of events that can be created */ public static final int MAX_EVENTS = 2000; diff --git a/src/main/java/net/floodlightcontroller/devicemanager/internal/DeviceManagerImpl.java b/src/main/java/net/floodlightcontroller/devicemanager/internal/DeviceManagerImpl.java index 8586501d4..7b8786924 100755 --- a/src/main/java/net/floodlightcontroller/devicemanager/internal/DeviceManagerImpl.java +++ b/src/main/java/net/floodlightcontroller/devicemanager/internal/DeviceManagerImpl.java @@ -53,6 +53,7 @@ import net.floodlightcontroller.core.module.IFloodlightService; import net.floodlightcontroller.core.util.ListenerDispatcher; import net.floodlightcontroller.core.util.SingletonTask; import net.floodlightcontroller.debugcounter.IDebugCounterService; +import net.floodlightcontroller.debugcounter.IDebugCounterService.MaxCountersRegistered; import net.floodlightcontroller.debugcounter.NullDebugCounter; import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType; import net.floodlightcontroller.devicemanager.IDevice; @@ -113,7 +114,6 @@ IFlowReconcileListener, IInfoProvider { protected static Logger logger = LoggerFactory.getLogger(DeviceManagerImpl.class); - public static final String MODULE_NAME = "devicemanager"; protected IFloodlightProviderService floodlightProvider; protected ITopologyService topology; @@ -127,66 +127,38 @@ IFlowReconcileListener, IInfoProvider { private IStoreClient<String,DeviceSyncRepresentation> storeClient; private DeviceSyncManager deviceSyncManager; - public static final String CNT_INCOMING = MODULE_NAME + "-incoming"; - public static final String CNT_RECONCILE_REQUEST = MODULE_NAME + - "-reconcileRequest"; - public static final String CNT_RECONCILE_NO_SOURCE = MODULE_NAME + - "-reconcileNoSourceDevice"; - public static final String CNT_RECONCILE_NO_DEST = MODULE_NAME + - "-reconcileNoDestDevice"; - public static final String CNT_BROADCAST_SOURCE = MODULE_NAME + - "-broadcastSource"; - public static final String CNT_NO_SOURCE = MODULE_NAME + - "-noSourceDevice"; - public static final String CNT_NO_DEST = MODULE_NAME + - "-noDestDevice"; - public static final String CNT_DHCP_CLIENT_NAME_SNOOPED = MODULE_NAME + - "-dhcpClientNameSnooped"; - public static final String CNT_DEVICE_ON_INTERAL_PORT_NOT_LEARNED = - MODULE_NAME + "-deviceOnInternalPortNotLearned"; - public static final String CNT_PACKET_NOT_ALLOWED = MODULE_NAME + - "-packetNotAllowed"; - public static final String CNT_NEW_DEVICE = MODULE_NAME + - "-newDevice"; - public static final String CNT_PACKET_ON_INTERNAL_PORT_FOR_KNOWN_DEVICE = - MODULE_NAME + "-packetOnInternalPortForKnownDevice"; - public static final String CNT_NEW_ENTITY = MODULE_NAME + - "-newEntity"; - public static final String CNT_DEVICE_CHANGED = MODULE_NAME + - "-deviceChanged"; - public static final String CNT_DEVICE_MOVED = MODULE_NAME + - "-deviceMoved"; - public static final String CNT_CLEANUP_ENTITIES_RUNS = MODULE_NAME + - "-cleanupEntitiesRuns"; - public static final String CNT_ENTITY_REMOVED_TIMEOUT = MODULE_NAME + - "-entityRemovedTimeout"; - public static final String CNT_DEVICE_DELETED = MODULE_NAME + - "-deviceDeleted"; - public static final String CNT_DEVICE_RECLASSIFY_DELETE = MODULE_NAME + - "-deviceReclassifyDelete"; - public static final String CNT_DEVICE_STORED = MODULE_NAME + - "-deviceStored"; - public static final String CNT_DEVICE_STORE_THROTTLED = MODULE_NAME + - "-deviceStoreThrottled"; - public static final String CNT_DEVICE_REMOVED_FROM_STORE = MODULE_NAME + - "-deviceRemovedFromStore"; - public static final String CNT_SYNC_EXCEPTION = MODULE_NAME + - "-syncException"; - public static final String CNT_DEVICES_FROM_STORE = MODULE_NAME + - "-devicesFromStore"; - public static final String CNT_CONSOLIDATE_STORE_RUNS = MODULE_NAME + - "-consolidateStoreRuns"; - public static final String CNT_CONSOLIDATE_STORE_DEVICES_REMOVED = MODULE_NAME + - "-consolidateStoreDevicesRemoved"; - public static final String CNT_TRANSITION_TO_MASTER = MODULE_NAME + - "-transitionToMaster"; - - - - - - - + /** + * Debug Counters + */ + public static final String MODULE_NAME = "devicemanager"; + public static final String WARN = "warn"; + public static int CNT_INCOMING; + public static int CNT_RECONCILE_REQUEST; + public static int CNT_RECONCILE_NO_SOURCE; + public static int CNT_RECONCILE_NO_DEST; + public static int CNT_BROADCAST_SOURCE; + public static int CNT_NO_SOURCE; + public static int CNT_NO_DEST; + public static int CNT_DHCP_CLIENT_NAME_SNOOPED; + public static int CNT_DEVICE_ON_INTERAL_PORT_NOT_LEARNED; + public static int CNT_PACKET_NOT_ALLOWED; + public static int CNT_NEW_DEVICE; + public static int CNT_PACKET_ON_INTERNAL_PORT_FOR_KNOWN_DEVICE; + public static int CNT_NEW_ENTITY; + public static int CNT_DEVICE_CHANGED; + public static int CNT_DEVICE_MOVED; + public static int CNT_CLEANUP_ENTITIES_RUNS; + public static int CNT_ENTITY_REMOVED_TIMEOUT; + public static int CNT_DEVICE_DELETED; + public static int CNT_DEVICE_RECLASSIFY_DELETE; + public static int CNT_DEVICE_STORED; + public static int CNT_DEVICE_STORE_THROTTLED; + public static int CNT_DEVICE_REMOVED_FROM_STORE; + public static int CNT_SYNC_EXCEPTION; + public static int CNT_DEVICES_FROM_STORE; + public static int CNT_CONSOLIDATE_STORE_RUNS; + public static int CNT_CONSOLIDATE_STORE_DEVICES_REMOVED; + public static int CNT_TRANSITION_TO_MASTER; private boolean isMaster = false; @@ -725,7 +697,7 @@ IFlowReconcileListener, IInfoProvider { FloodlightContext cntx) { switch (msg.getType()) { case PACKET_IN: - debugCounters.updateCounter(CNT_INCOMING); + debugCounters.updateCounter(CNT_INCOMING, false); return this.processPacketInMessage(sw, (OFPacketIn) msg, cntx); default: @@ -757,19 +729,19 @@ IFlowReconcileListener, IInfoProvider { } protected Command reconcileFlow(OFMatchReconcile ofm) { - debugCounters.updateCounter(CNT_RECONCILE_REQUEST); + debugCounters.updateCounter(CNT_RECONCILE_REQUEST, false); // Extract source entity information Entity srcEntity = getEntityFromFlowMod(ofm.ofmWithSwDpid, true); if (srcEntity == null) { - debugCounters.updateCounter(CNT_RECONCILE_NO_SOURCE); + debugCounters.updateCounter(CNT_RECONCILE_NO_SOURCE, false); return Command.STOP; } // Find the device by source entity Device srcDevice = findDeviceByEntity(srcEntity); if (srcDevice == null) { - debugCounters.updateCounter(CNT_RECONCILE_NO_SOURCE); + debugCounters.updateCounter(CNT_RECONCILE_NO_SOURCE, false); return Command.STOP; } // Store the source device in the context @@ -784,9 +756,9 @@ IFlowReconcileListener, IInfoProvider { if (dstDevice != null) fcStore.put(ofm.cntx, CONTEXT_DST_DEVICE, dstDevice); else - debugCounters.updateCounter(CNT_RECONCILE_NO_DEST); + debugCounters.updateCounter(CNT_RECONCILE_NO_DEST, false); } else { - debugCounters.updateCounter(CNT_RECONCILE_NO_DEST); + debugCounters.updateCounter(CNT_RECONCILE_NO_DEST, false); } if (logger.isTraceEnabled()) { logger.trace("Reconciling flow: match={}, srcEntity={}, srcDev={}, " @@ -890,7 +862,6 @@ IFlowReconcileListener, IInfoProvider { cleanupEntities(); entityCleanupTask.reschedule(ENTITY_CLEANUP_INTERVAL, TimeUnit.SECONDS); - debugCounters.flushCounters(); } }; entityCleanupTask = new SingletonTask(ses, ecr); @@ -903,7 +874,6 @@ IFlowReconcileListener, IInfoProvider { deviceSyncManager.consolidateStore(); storeConsolidateTask.reschedule(syncStoreConsolidateIntervalMs, TimeUnit.MILLISECONDS); - debugCounters.flushCounters(); } }; storeConsolidateTask = new SingletonTask(ses, consolidateStoreRunner); @@ -937,114 +907,138 @@ IFlowReconcileListener, IInfoProvider { debugCounters = new NullDebugCounter(); return; } - debugCounters.registerCounter(CNT_INCOMING, - "All incoming packets seen by this module", CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_RECONCILE_REQUEST, - "Number of flows that have been received for reconciliation by " + - "this module", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_RECONCILE_NO_SOURCE, - "Number of flow reconcile events that failed because no source " + - "device could be identified", - CounterType.WARN); // is this really a warning - debugCounters.registerCounter(CNT_RECONCILE_NO_DEST, - "Number of flow reconcile events that failed because no " + - "destination device could be identified", - CounterType.WARN); // is this really a warning - debugCounters.registerCounter(CNT_BROADCAST_SOURCE, - "Number of packetIns that were discarded because the source " + - "MAC was broadcast or multicast", - CounterType.WARN); - debugCounters.registerCounter(CNT_NO_SOURCE, - "Number of packetIns that were discarded because the " + - "could not identify a source device. This can happen if a " + - "packet is not allowed, appears on an illegal port, does not " + - "have a valid address space, etc.", - CounterType.WARN); - debugCounters.registerCounter(CNT_NO_DEST, - "Number of packetIns that did not have an associated " + - "destination device. E.g., because the destination MAC is " + - "broadcast/multicast or is not yet known to the controller.", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DHCP_CLIENT_NAME_SNOOPED, - "Number of times a DHCP client name was snooped from a " + - "packetIn.", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_ON_INTERAL_PORT_NOT_LEARNED, - "Number of times packetIn was received on an internal port and" + - "no source device is known for the source MAC. The packetIn is " + - "discarded.", - CounterType.WARN); - debugCounters.registerCounter(CNT_PACKET_NOT_ALLOWED, - "Number of times a packetIn was not allowed due to spoofing " + - "protection configuration.", - CounterType.WARN); // is this really a warning? - debugCounters.registerCounter(CNT_NEW_DEVICE, - "Number of times a new device was learned", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_PACKET_ON_INTERNAL_PORT_FOR_KNOWN_DEVICE, - "Number of times a packetIn was received on an internal port " + - "for a known device.", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_NEW_ENTITY, - "Number of times a new entity was learned for an existing device", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_CHANGED, - "Number of times device properties have changed", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_MOVED, - "Number of times devices have moved", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_CLEANUP_ENTITIES_RUNS, - "Number of times the entity cleanup task has been run", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_ENTITY_REMOVED_TIMEOUT, - "Number of times entities have been removed due to timeout " + - "(entity has been inactive for " + ENTITY_TIMEOUT/1000 + "s)", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_DELETED, - "Number of devices that have been removed due to inactivity", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_RECLASSIFY_DELETE, - "Number of devices that required reclassification and have been " + - "temporarily delete for reclassification", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_STORED, - "Number of device entries written or updated to the sync store", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_STORE_THROTTLED, - "Number of times a device update to the sync store was " + - "requested but not performed because the same device entities " + - "have recently been updated already", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_DEVICE_REMOVED_FROM_STORE, - "Number of devices that were removed from the sync store " + - "because the local controller removed the device due to " + - "inactivity", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_SYNC_EXCEPTION, - "Number of times an operation on the sync store resulted in " + - "sync exception", - CounterType.WARN); // it this an error? - debugCounters.registerCounter(CNT_DEVICES_FROM_STORE, - "Number of devices that were read from the sync store after " + - "the local controller transitioned from SLAVE to MASTER", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_CONSOLIDATE_STORE_RUNS, - "Number of times the task to consolidate entries in the " + - "store witch live known devices has been run", - CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(CNT_CONSOLIDATE_STORE_DEVICES_REMOVED, - "Number of times a device has been removed from the sync " + - "store because no corresponding live device is known. " + - "This indicates a remote controller still writing device " + - "entries despite the local controller being MASTER or an " + - "incosistent store update from the local controller.", - CounterType.WARN); - debugCounters.registerCounter(CNT_TRANSITION_TO_MASTER, - "Number of times this controller has transitioned from SLAVE " + - "to MASTER role. Will be 0 or 1.", - CounterType.ALWAYS_COUNT); + try { + CNT_INCOMING = debugCounters.registerCounter(MODULE_NAME, "incoming", + "All incoming packets seen by this module", CounterType.ALWAYS_COUNT, + new Object[] {}); + CNT_RECONCILE_REQUEST = debugCounters.registerCounter(MODULE_NAME, + "reconcile-request", + "Number of flows that have been received for reconciliation by " + + "this module", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_RECONCILE_NO_SOURCE = debugCounters.registerCounter(MODULE_NAME, + "reconcile-no-source-device", + "Number of flow reconcile events that failed because no source " + + "device could be identified", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); // is this really a warning + CNT_RECONCILE_NO_DEST = debugCounters.registerCounter(MODULE_NAME, + "reconcile-no-dest-device", + "Number of flow reconcile events that failed because no " + + "destination device could be identified", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); // is this really a warning + CNT_BROADCAST_SOURCE = debugCounters.registerCounter(MODULE_NAME, + "broadcast-source", + "Number of packetIns that were discarded because the source " + + "MAC was broadcast or multicast", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); + CNT_NO_SOURCE = debugCounters.registerCounter(MODULE_NAME, "no-source-device", + "Number of packetIns that were discarded because the " + + "could not identify a source device. This can happen if a " + + "packet is not allowed, appears on an illegal port, does not " + + "have a valid address space, etc.", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); + CNT_NO_DEST = debugCounters.registerCounter(MODULE_NAME, "no-dest-device", + "Number of packetIns that did not have an associated " + + "destination device. E.g., because the destination MAC is " + + "broadcast/multicast or is not yet known to the controller.", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DHCP_CLIENT_NAME_SNOOPED = debugCounters.registerCounter(MODULE_NAME, + "dhcp-client-name-snooped", + "Number of times a DHCP client name was snooped from a " + + "packetIn.", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_ON_INTERAL_PORT_NOT_LEARNED = debugCounters.registerCounter( + MODULE_NAME, + "device-on-internal-port-not-learned", + "Number of times packetIn was received on an internal port and" + + "no source device is known for the source MAC. The packetIn is " + + "discarded.", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); + CNT_PACKET_NOT_ALLOWED = debugCounters.registerCounter(MODULE_NAME, + "packet-not-allowed", + "Number of times a packetIn was not allowed due to spoofing " + + "protection configuration.", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); // is this really a warning? + CNT_NEW_DEVICE = debugCounters.registerCounter(MODULE_NAME, "new-device", + "Number of times a new device was learned", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_PACKET_ON_INTERNAL_PORT_FOR_KNOWN_DEVICE = debugCounters.registerCounter( + MODULE_NAME, + "packet-on-internal-port-for-known-device", + "Number of times a packetIn was received on an internal port " + + "for a known device.", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_NEW_ENTITY = debugCounters.registerCounter(MODULE_NAME, "new-entity", + "Number of times a new entity was learned for an existing device", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_CHANGED = debugCounters.registerCounter(MODULE_NAME, "device-changed", + "Number of times device properties have changed", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_MOVED = debugCounters.registerCounter(MODULE_NAME, "device-moved", + "Number of times devices have moved", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_CLEANUP_ENTITIES_RUNS = debugCounters.registerCounter(MODULE_NAME, + "cleanup-entities-runs", + "Number of times the entity cleanup task has been run", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_ENTITY_REMOVED_TIMEOUT = debugCounters.registerCounter(MODULE_NAME, + "entity-removed-timeout", + "Number of times entities have been removed due to timeout " + + "(entity has been inactive for " + ENTITY_TIMEOUT/1000 + "s)", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_DELETED = debugCounters.registerCounter(MODULE_NAME, "device-deleted", + "Number of devices that have been removed due to inactivity", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_RECLASSIFY_DELETE = debugCounters.registerCounter(MODULE_NAME, + "device-reclassify-delete", + "Number of devices that required reclassification and have been " + + "temporarily delete for reclassification", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_STORED = debugCounters.registerCounter(MODULE_NAME, "device-stored", + "Number of device entries written or updated to the sync store", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_STORE_THROTTLED = debugCounters.registerCounter(MODULE_NAME, + "device-store-throttled", + "Number of times a device update to the sync store was " + + "requested but not performed because the same device entities " + + "have recently been updated already", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_DEVICE_REMOVED_FROM_STORE = debugCounters.registerCounter(MODULE_NAME, + "device-removed-from-store", + "Number of devices that were removed from the sync store " + + "because the local controller removed the device due to " + + "inactivity", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_SYNC_EXCEPTION = debugCounters.registerCounter(MODULE_NAME, "sync-exception", + "Number of times an operation on the sync store resulted in " + + "sync exception", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); // it this an error? + CNT_DEVICES_FROM_STORE = debugCounters.registerCounter(MODULE_NAME, + "devices-from-store", + "Number of devices that were read from the sync store after " + + "the local controller transitioned from SLAVE to MASTER", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_CONSOLIDATE_STORE_RUNS = debugCounters.registerCounter(MODULE_NAME, + "consolidate-store-runs", + "Number of times the task to consolidate entries in the " + + "store witch live known devices has been run", + CounterType.ALWAYS_COUNT, new Object[] {}); + CNT_CONSOLIDATE_STORE_DEVICES_REMOVED = debugCounters.registerCounter(MODULE_NAME, + "consolidate-store-devices-removed", + "Number of times a device has been removed from the sync " + + "store because no corresponding live device is known. " + + "This indicates a remote controller still writing device " + + "entries despite the local controller being MASTER or an " + + "incosistent store update from the local controller.", + CounterType.ALWAYS_COUNT, new Object[] {WARN}); + CNT_TRANSITION_TO_MASTER = debugCounters.registerCounter(MODULE_NAME, + "transition-to-master", + "Number of times this controller has transitioned from SLAVE " + + "to MASTER role. Will be 0 or 1.", + CounterType.ALWAYS_COUNT, new Object[] {}); + } catch (MaxCountersRegistered e) { + e.printStackTrace(); + } } // *************** @@ -1101,7 +1095,7 @@ IFlowReconcileListener, IInfoProvider { Entity srcEntity = getSourceEntityFromPacket(eth, sw.getId(), pi.getInPort()); if (srcEntity == null) { - debugCounters.updateCounter(CNT_BROADCAST_SOURCE); + debugCounters.updateCounter(CNT_BROADCAST_SOURCE, false); return Command.STOP; } @@ -1116,7 +1110,7 @@ IFlowReconcileListener, IInfoProvider { // Learn/lookup device information Device srcDevice = learnDeviceByEntity(srcEntity); if (srcDevice == null) { - debugCounters.updateCounter(CNT_NO_SOURCE); + debugCounters.updateCounter(CNT_NO_SOURCE, false); return Command.STOP; } @@ -1133,9 +1127,9 @@ IFlowReconcileListener, IInfoProvider { if (dstDevice != null) fcStore.put(cntx, CONTEXT_DST_DEVICE, dstDevice); else - debugCounters.updateCounter(CNT_NO_DEST); + debugCounters.updateCounter(CNT_NO_DEST, false); } else { - debugCounters.updateCounter(CNT_NO_DEST); + debugCounters.updateCounter(CNT_NO_DEST, false); } if (logger.isTraceEnabled()) { @@ -1170,7 +1164,7 @@ IFlowReconcileListener, IInfoProvider { DHCPOption dhcpOption = dhcp.getOption( DHCPOptionCode.OptionCode_Hostname); if (dhcpOption != null) { - debugCounters.updateCounter(CNT_DHCP_CLIENT_NAME_SNOOPED); + debugCounters.updateCounter(CNT_DHCP_CLIENT_NAME_SNOOPED, false); srcDevice.dhcpClientName = new String(dhcpOption.getData()); } } @@ -1333,7 +1327,8 @@ IFlowReconcileListener, IInfoProvider { inPort = ofmWithSwDpid.getOfMatch().getInputPort(); } - /**for the new flow cache design, the flow mods retrived are not always from the source, learn AP should be disabled --meiyang*/ + /**for the new flow cache design, the flow mods retrived are not always + * from the source, learn AP should be disabled --meiyang*/ boolean learnap = false; /** * if (swDpid == null || @@ -1510,7 +1505,8 @@ IFlowReconcileListener, IInfoProvider { if (entity.hasSwitchPort() && !topology.isAttachmentPointPort(entity.getSwitchDPID(), entity.getSwitchPort().shortValue())) { - debugCounters.updateCounter(CNT_DEVICE_ON_INTERAL_PORT_NOT_LEARNED); + debugCounters.updateCounter(CNT_DEVICE_ON_INTERAL_PORT_NOT_LEARNED, + false); if (logger.isDebugEnabled()) { logger.debug("Not learning new device on internal" + " link: {}", entity); @@ -1521,7 +1517,7 @@ IFlowReconcileListener, IInfoProvider { // Before we create the new device also check if // the entity is allowed (e.g., for spoofing protection) if (!isEntityAllowed(entity, entityClass)) { - debugCounters.updateCounter(CNT_PACKET_NOT_ALLOWED); + debugCounters.updateCounter(CNT_PACKET_NOT_ALLOWED, false); if (logger.isDebugEnabled()) { logger.debug("PacketIn is not allowed {} {}", entityClass.getName(), entity); @@ -1551,7 +1547,7 @@ IFlowReconcileListener, IInfoProvider { // We need to count and log here. If we log earlier we could // hit a concurrent modification and restart the dev creation // and potentially count the device twice. - debugCounters.updateCounter(CNT_NEW_DEVICE); + debugCounters.updateCounter(CNT_NEW_DEVICE, false); if (logger.isDebugEnabled()) { logger.debug("New device created: {} deviceKey={}, entity={}", new Object[]{device, deviceKey, entity}); @@ -1565,7 +1561,7 @@ IFlowReconcileListener, IInfoProvider { } // if it gets here, we have a pre-existing Device for this Entity if (!isEntityAllowed(entity, device.getEntityClass())) { - debugCounters.updateCounter(CNT_PACKET_NOT_ALLOWED); + debugCounters.updateCounter(CNT_PACKET_NOT_ALLOWED, false); if (logger.isDebugEnabled()) { logger.info("PacketIn is not allowed {} {}", device.getEntityClass().getName(), entity); @@ -1578,7 +1574,8 @@ IFlowReconcileListener, IInfoProvider { if (entity.hasSwitchPort() && !topology.isAttachmentPointPort(entity.getSwitchDPID(), entity.getSwitchPort().shortValue())) { - debugCounters.updateCounter(CNT_PACKET_ON_INTERNAL_PORT_FOR_KNOWN_DEVICE); + debugCounters.updateCounter(CNT_PACKET_ON_INTERNAL_PORT_FOR_KNOWN_DEVICE, + false); break; } int entityindex = -1; @@ -1622,9 +1619,9 @@ IFlowReconcileListener, IInfoProvider { // We need to count here after all the possible "continue" // statements in this branch - debugCounters.updateCounter(CNT_NEW_ENTITY); + debugCounters.updateCounter(CNT_NEW_ENTITY, false); if (changedFields.size() > 0) { - debugCounters.updateCounter(CNT_DEVICE_CHANGED); + debugCounters.updateCounter(CNT_DEVICE_CHANGED, false); deviceUpdates = updateUpdates(deviceUpdates, new DeviceUpdate(newDevice, CHANGE, @@ -1885,7 +1882,7 @@ IFlowReconcileListener, IInfoProvider { * Clean up expired entities/devices */ protected void cleanupEntities () { - debugCounters.updateCounter(CNT_CLEANUP_ENTITIES_RUNS); + debugCounters.updateCounter(CNT_CLEANUP_ENTITIES_RUNS, true); Calendar c = Calendar.getInstance(); c.add(Calendar.MILLISECOND, -ENTITY_TIMEOUT); @@ -1918,7 +1915,7 @@ IFlowReconcileListener, IInfoProvider { break; } - debugCounters.updateCounter(CNT_ENTITY_REMOVED_TIMEOUT); + debugCounters.updateCounter(CNT_ENTITY_REMOVED_TIMEOUT, true); for (Entity e : toRemove) { removeEntity(e, d.getEntityClass(), d.getDeviceKey(), toKeep); } @@ -1954,7 +1951,7 @@ IFlowReconcileListener, IInfoProvider { if (update != null) { // need to count after all possibly continue stmts in // this branch - debugCounters.updateCounter(CNT_DEVICE_CHANGED); + debugCounters.updateCounter(CNT_DEVICE_CHANGED, true); deviceUpdates.add(update); } } else { @@ -1966,7 +1963,7 @@ IFlowReconcileListener, IInfoProvider { d = deviceMap.get(d.getDeviceKey()); if (null != d) continue; - debugCounters.updateCounter(CNT_DEVICE_DELETED); + debugCounters.updateCounter(CNT_DEVICE_DELETED, true); } deviceUpdates.add(update); } @@ -2134,7 +2131,7 @@ IFlowReconcileListener, IInfoProvider { * @param updates the updates to process. */ protected void sendDeviceMovedNotification(Device d) { - debugCounters.updateCounter(CNT_DEVICE_MOVED); + debugCounters.updateCounter(CNT_DEVICE_MOVED, false); deviceSyncManager.storeDevice(d); List<IDeviceListener> listeners = deviceListeners.getOrderedListeners(); if (listeners != null) { @@ -2193,7 +2190,7 @@ IFlowReconcileListener, IInfoProvider { return false; } - debugCounters.updateCounter(CNT_DEVICE_RECLASSIFY_DELETE); + debugCounters.updateCounter(CNT_DEVICE_RECLASSIFY_DELETE, false); LinkedList<DeviceUpdate> deviceUpdates = new LinkedList<DeviceUpdate>(); // delete this device and then re-learn all the entities @@ -2236,7 +2233,7 @@ IFlowReconcileListener, IInfoProvider { private class DeviceSyncManager { // maps (opaque) deviceKey to the time in System.nanoTime() when we // last wrote the device to the sync store - private ConcurrentMap<Long, Long> lastWriteTimes = + private final ConcurrentMap<Long, Long> lastWriteTimes = new ConcurrentHashMap<Long, Long>(); /** @@ -2275,7 +2272,7 @@ IFlowReconcileListener, IInfoProvider { writeUpdatedDeviceToStorage(d); lastWriteTimes.put(d.getDeviceKey(), now); } else { - debugCounters.updateCounter(CNT_DEVICE_STORE_THROTTLED); + debugCounters.updateCounter(CNT_DEVICE_STORE_THROTTLED, true); } } @@ -2296,12 +2293,12 @@ IFlowReconcileListener, IInfoProvider { // TODO: should probably do versioned delete. OTOH, even // if we accidentally delete, we'll write it again after // the next entity .... - debugCounters.updateCounter(CNT_DEVICE_REMOVED_FROM_STORE); + debugCounters.updateCounter(CNT_DEVICE_REMOVED_FROM_STORE, true); storeClient.delete(DeviceSyncRepresentation.computeKey(d)); } catch(ObsoleteVersionException e) { // FIXME } catch (SyncException e) { - debugCounters.updateCounter(CNT_SYNC_EXCEPTION); + debugCounters.updateCounter(CNT_SYNC_EXCEPTION, true); logger.error("Could not remove device " + d + " from store", e); } } @@ -2313,14 +2310,14 @@ IFlowReconcileListener, IInfoProvider { */ private void removeDevice(Versioned<DeviceSyncRepresentation> dev) { try { - debugCounters.updateCounter(CNT_DEVICE_REMOVED_FROM_STORE); + debugCounters.updateCounter(CNT_DEVICE_REMOVED_FROM_STORE, true); storeClient.delete(dev.getValue().getKey(), dev.getVersion()); } catch(ObsoleteVersionException e) { // Key was locally modified by another thread. // Do not delete and ignore. } catch(SyncException e) { - debugCounters.updateCounter(CNT_SYNC_EXCEPTION); + debugCounters.updateCounter(CNT_SYNC_EXCEPTION, true); logger.error("Failed to remove device entry for " + dev.toString() + " from store.", e); } @@ -2334,13 +2331,13 @@ IFlowReconcileListener, IInfoProvider { if (logger.isDebugEnabled()) { logger.debug("Transitioning to MASTER role"); } - debugCounters.updateCounter(CNT_TRANSITION_TO_MASTER); + debugCounters.updateCounter(CNT_TRANSITION_TO_MASTER, true); IClosableIterator<Map.Entry<String,Versioned<DeviceSyncRepresentation>>> iter = null; try { iter = storeClient.entries(); } catch (SyncException e) { - debugCounters.updateCounter(CNT_SYNC_EXCEPTION); + debugCounters.updateCounter(CNT_SYNC_EXCEPTION, true); logger.error("Failed to read devices from sync store", e); return; } @@ -2352,7 +2349,7 @@ IFlowReconcileListener, IInfoProvider { versionedDevice.getValue(); if (storedDevice == null) continue; - debugCounters.updateCounter(CNT_DEVICES_FROM_STORE); + debugCounters.updateCounter(CNT_DEVICES_FROM_STORE, true); for(SyncEntity se: storedDevice.getEntities()) { learnDeviceByEntity(se.asEntity()); } @@ -2372,7 +2369,7 @@ IFlowReconcileListener, IInfoProvider { */ private void writeUpdatedDeviceToStorage(Device device) { try { - debugCounters.updateCounter(CNT_DEVICE_STORED); + debugCounters.updateCounter(CNT_DEVICE_STORED, true); // FIXME: use a versioned put DeviceSyncRepresentation storeDevice = new DeviceSyncRepresentation(device); @@ -2381,7 +2378,7 @@ IFlowReconcileListener, IInfoProvider { // FIXME: what's the right behavior here. Can the store client // even throw this error? } catch (SyncException e) { - debugCounters.updateCounter(CNT_SYNC_EXCEPTION); + debugCounters.updateCounter(CNT_SYNC_EXCEPTION, true); logger.error("Could not write device " + device + " to sync store:", e); } @@ -2405,7 +2402,7 @@ IFlowReconcileListener, IInfoProvider { private void consolidateStore() { if (!isMaster) return; - debugCounters.updateCounter(CNT_CONSOLIDATE_STORE_RUNS); + debugCounters.updateCounter(CNT_CONSOLIDATE_STORE_RUNS, true); if (logger.isDebugEnabled()) { logger.debug("Running consolidateStore."); } @@ -2414,7 +2411,7 @@ IFlowReconcileListener, IInfoProvider { try { iter = storeClient.entries(); } catch (SyncException e) { - debugCounters.updateCounter(CNT_SYNC_EXCEPTION); + debugCounters.updateCounter(CNT_SYNC_EXCEPTION, true); logger.error("Failed to read devices from sync store", e); return; } @@ -2451,7 +2448,7 @@ IFlowReconcileListener, IInfoProvider { + "corresponding live device", storedDevice.getKey()); } - debugCounters.updateCounter(CNT_CONSOLIDATE_STORE_DEVICES_REMOVED); + debugCounters.updateCounter(CNT_CONSOLIDATE_STORE_DEVICES_REMOVED, true); removeDevice(versionedDevice); } } diff --git a/src/main/java/net/floodlightcontroller/linkdiscovery/internal/LinkDiscoveryManager.java b/src/main/java/net/floodlightcontroller/linkdiscovery/internal/LinkDiscoveryManager.java index 7013ae151..a8b599530 100644 --- a/src/main/java/net/floodlightcontroller/linkdiscovery/internal/LinkDiscoveryManager.java +++ b/src/main/java/net/floodlightcontroller/linkdiscovery/internal/LinkDiscoveryManager.java @@ -59,6 +59,7 @@ import net.floodlightcontroller.core.module.IFloodlightService; import net.floodlightcontroller.core.util.SingletonTask; import net.floodlightcontroller.debugcounter.IDebugCounterService; import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType; +import net.floodlightcontroller.debugcounter.IDebugCounterService.MaxCountersRegistered; import net.floodlightcontroller.debugcounter.NullDebugCounter; import net.floodlightcontroller.debugevent.IDebugEventService; import net.floodlightcontroller.debugevent.IDebugEventService.MaxEventsRegistered; @@ -260,6 +261,16 @@ public class LinkDiscoveryManager implements IOFMessageListener, private IHAListener haListener; + /** + * Debug Counter Ids + */ + private int QUARANTINE_DROPS; + private int IGNORE_SRC_MAC_DROPS; + private int INCOMING; + private int LINKLOCAL_DROPS; + private int LLDP_EOL; + + //********************* // ILinkDiscoveryService //********************* @@ -511,7 +522,7 @@ public class LinkDiscoveryManager implements IOFMessageListener, FloodlightContext cntx) { switch (msg.getType()) { case PACKET_IN: - debugCounters.updateCounter("linkdiscovery-incoming"); + debugCounters.updateCounter(INCOMING, false); return this.handlePacketIn(sw.getId(), (OFPacketIn) msg, cntx); default: @@ -553,7 +564,7 @@ public class LinkDiscoveryManager implements IOFMessageListener, } else if (eth.getEtherType() < 1500) { long destMac = eth.getDestinationMAC().toLong(); if ((destMac & LINK_LOCAL_MASK) == LINK_LOCAL_VALUE) { - debugCounters.updateCounter("linkdiscovery-linklocaldrops"); + debugCounters.updateCounter(LINKLOCAL_DROPS, false); if (log.isTraceEnabled()) { log.trace("Ignoring packet addressed to 802.1D/Q " + "reserved address."); @@ -563,14 +574,14 @@ public class LinkDiscoveryManager implements IOFMessageListener, } if (ignorePacketInFromSource(eth.getSourceMAC().toLong())) { - debugCounters.updateCounter("linkdiscovery-ignoresrcmacdrops"); + debugCounters.updateCounter(IGNORE_SRC_MAC_DROPS, false); return Command.STOP; } // If packet-in is from a quarantine port, stop processing. NodePortTuple npt = new NodePortTuple(sw, pi.getInPort()); if (quarantineQueue.contains(npt)) { - debugCounters.updateCounter("linkdiscovery-quarantinedrops"); + debugCounters.updateCounter(QUARANTINE_DROPS, false); return Command.STOP; } @@ -766,7 +777,7 @@ public class LinkDiscoveryManager implements IOFMessageListener, removeFromMaintenanceQueue(nptDst); // Consume this message - debugCounters.updateCounter("linkdiscovery-lldpeol"); + debugCounters.updateCounter(LLDP_EOL, false); return Command.STOP; } @@ -2103,19 +2114,24 @@ public class LinkDiscoveryManager implements IOFMessageListener, debugCounters = new NullDebugCounter(); return; } - debugCounters.registerCounter(getName() + "-" + "incoming", - "All incoming packets seen by this module", CounterType.ALWAYS_COUNT); - debugCounters.registerCounter(getName() + "-" + "lldpeol", - "End of Life for LLDP packets", CounterType.COUNT_ON_DEMAND); - debugCounters.registerCounter(getName() + "-" + "linklocaldrops", - "All link local packets dropped by this module", - CounterType.COUNT_ON_DEMAND); - debugCounters.registerCounter(getName() + "-" + "ignoresrcmacdrops", - "All packets whose srcmac is configured to be dropped by this module", - CounterType.COUNT_ON_DEMAND); - debugCounters.registerCounter(getName() + "-" + "quarantinedrops", - "All packets arriving on qurantined ports dropped by this module", - CounterType.COUNT_ON_DEMAND); + try { + INCOMING = debugCounters.registerCounter(getName(), "incoming", + "All incoming packets seen by this module", CounterType.ALWAYS_COUNT, + new Object[] {}); + LLDP_EOL = debugCounters.registerCounter(getName(), "lldp-eol", + "End of Life for LLDP packets", CounterType.COUNT_ON_DEMAND, new Object[] {}); + LINKLOCAL_DROPS = debugCounters.registerCounter(getName(), "linklocal-drops", + "All link local packets dropped by this module", + CounterType.COUNT_ON_DEMAND, new Object[] {}); + IGNORE_SRC_MAC_DROPS = debugCounters.registerCounter(getName(), "ignore-srcmac-drops", + "All packets whose srcmac is configured to be dropped by this module", + CounterType.COUNT_ON_DEMAND, new Object[] {}); + QUARANTINE_DROPS = debugCounters.registerCounter(getName(), "quarantine-drops", + "All packets arriving on quarantined ports dropped by this module", + CounterType.COUNT_ON_DEMAND, new Object[] {}); + } catch (MaxCountersRegistered e) { + e.printStackTrace(); + } } private void registerLinkDiscoveryDebugEvents() { diff --git a/src/main/java/net/floodlightcontroller/topology/TopologyManager.java b/src/main/java/net/floodlightcontroller/topology/TopologyManager.java index fee256f55..4dc44cad8 100644 --- a/src/main/java/net/floodlightcontroller/topology/TopologyManager.java +++ b/src/main/java/net/floodlightcontroller/topology/TopologyManager.java @@ -48,6 +48,7 @@ import net.floodlightcontroller.core.util.SingletonTask; import net.floodlightcontroller.counter.ICounterStoreService; import net.floodlightcontroller.debugcounter.IDebugCounterService; import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType; +import net.floodlightcontroller.debugcounter.IDebugCounterService.MaxCountersRegistered; import net.floodlightcontroller.debugcounter.NullDebugCounter; import net.floodlightcontroller.linkdiscovery.ILinkDiscoveryListener; import net.floodlightcontroller.linkdiscovery.ILinkDiscoveryService; @@ -156,6 +157,11 @@ public class TopologyManager implements private IHAListener haListener; + /** + * Debug Counters + */ + protected static int INCOMING; + // Getter/Setter methods /** * Get the time interval for the period topology updates, if any. @@ -654,7 +660,7 @@ public class TopologyManager implements FloodlightContext cntx) { switch (msg.getType()) { case PACKET_IN: - debugCounters.updateCounter("topology-incoming"); + debugCounters.updateCounter(INCOMING, false); return this.processPacketInMessage(sw, (OFPacketIn) msg, cntx); default: @@ -791,8 +797,13 @@ public class TopologyManager implements debugCounters = new NullDebugCounter(); return; } - debugCounters.registerCounter(getName() + "-" + "incoming", - "All incoming packets seen by this module", CounterType.ALWAYS_COUNT); + try { + INCOMING = debugCounters.registerCounter(getName(), "incoming", + "All incoming packets seen by this module", CounterType.ALWAYS_COUNT, + new Object[] {}); + } catch (MaxCountersRegistered e) { + e.printStackTrace(); + } } protected void addRestletRoutable() { diff --git a/src/main/java/org/sdnplatform/sync/internal/SyncManager.java b/src/main/java/org/sdnplatform/sync/internal/SyncManager.java index 913cbc926..717d826e2 100644 --- a/src/main/java/org/sdnplatform/sync/internal/SyncManager.java +++ b/src/main/java/org/sdnplatform/sync/internal/SyncManager.java @@ -60,6 +60,7 @@ import net.floodlightcontroller.core.module.IFloodlightService; import net.floodlightcontroller.core.util.SingletonTask; import net.floodlightcontroller.debugcounter.IDebugCounterService; import net.floodlightcontroller.debugcounter.IDebugCounterService.CounterType; +import net.floodlightcontroller.debugcounter.IDebugCounterService.MaxCountersRegistered; import net.floodlightcontroller.storage.IStorageSourceService; import net.floodlightcontroller.threadpool.IThreadPoolService; @@ -77,18 +78,18 @@ public class SyncManager extends AbstractSyncManager { protected IThreadPoolService threadPool; protected IDebugCounterService debugCounter; - + /** - * The store registry holds the storage engines that provide + * The store registry holds the storage engines that provide * access to the data */ private StoreRegistry storeRegistry = null; - + private IClusterConfigProvider clusterConfigProvider; private ClusterConfig clusterConfig = new ClusterConfig(); protected RPCService rpcService = null; - + /** * Interval between cleanup tasks in seconds */ @@ -103,12 +104,12 @@ public class SyncManager extends AbstractSyncManager { * Interval between configuration rescans */ private static final int CONFIG_RESCAN_INTERVAL = 10; - + /** * Task for performing periodic maintenance/cleanup on local stores */ private SingletonTask cleanupTask; - + /** * Task for periodic antientropy between nodes */ @@ -118,13 +119,13 @@ public class SyncManager extends AbstractSyncManager { * Task to periodically rescan configuration */ private SingletonTask updateConfigTask; - + /** - * Number of {@link HintWorker} workers used to drain the queue of writes + * Number of {@link HintWorker} workers used to drain the queue of writes * that need to be sent to the connected nodes */ private static final int SYNC_WORKER_POOL = 2; - + /** * A thread pool for the {@link HintWorker} threads. */ @@ -133,32 +134,34 @@ public class SyncManager extends AbstractSyncManager { /** * Random number generator */ - private Random random = new Random(); + private final Random random = new Random(); /** * A map of the currently-allocated cursors */ - private Map<Integer, Cursor> cursorMap = - new ConcurrentHashMap<Integer, Cursor>(); - + private final Map<Integer, Cursor> cursorMap = + new ConcurrentHashMap<Integer, Cursor>(); + /** - * Whether to allow persistent stores or to use in-memory even + * Whether to allow persistent stores or to use in-memory even * when persistence is requested */ private boolean persistenceEnabled = true; - - private static final String PACKAGE = + + private static final String PACKAGE = ISyncService.class.getPackage().getName(); - public static final String COUNTER_HINTS = PACKAGE + "-hints"; - public static final String COUNTER_SENT_VALUES = PACKAGE + "-sent_values"; - public static final String COUNTER_RECEIVED_VALUES = - PACKAGE + "-received_values"; - public static final String COUNTER_PUTS = PACKAGE + "-puts"; - public static final String COUNTER_GETS = PACKAGE + "-gets"; - public static final String COUNTER_ITERATORS = PACKAGE + "-iterators"; - public static final String COUNTER_ERROR_REMOTE = PACKAGE + "-error-remote"; - public static final String COUNTER_ERROR_PROCESSING = - PACKAGE + "-error-processing"; + + /** + * Debug Counters + */ + public static int COUNTER_HINTS; + public static int COUNTER_SENT_VALUES; + public static int COUNTER_RECEIVED_VALUES; + public static int COUNTER_PUTS; + public static int COUNTER_GETS; + public static int COUNTER_ITERATORS; + public static int COUNTER_ERROR_REMOTE; + public static int COUNTER_ERROR_PROCESSING; // ************ // ISyncService @@ -175,7 +178,7 @@ public class SyncManager extends AbstractSyncManager { } @Override - public void registerPersistentStore(String storeName, Scope scope) + public void registerPersistentStore(String storeName, Scope scope) throws PersistException { storeRegistry.register(storeName, scope, persistenceEnabled); } @@ -197,11 +200,11 @@ public class SyncManager extends AbstractSyncManager { * Perform periodic scheduled cleanup. Note that this will be called * automatically and you shouldn't generally call it directly except for * testing - * @throws SyncException + * @throws SyncException */ public void cleanup() throws SyncException { for (SynchronizingStorageEngine store : storeRegistry.values()) { - store.cleanupTask(); + store.cleanupTask(); } } @@ -214,34 +217,34 @@ public class SyncManager extends AbstractSyncManager { public void antientropy(Node node) { if (!rpcService.isConnected(node.getNodeId())) return; - logger.info("[{}->{}] Synchronizing local state to remote node", + logger.info("[{}->{}] Synchronizing local state to remote node", getLocalNodeId(), node.getNodeId()); for (SynchronizingStorageEngine store : storeRegistry.values()) { if (Scope.LOCAL.equals(store.getScope())) { - if (node.getDomainId() != + if (node.getDomainId() != getClusterConfig().getNode().getDomainId()) continue; } else if (Scope.UNSYNCHRONIZED.equals(store.getScope())) { continue; } - - IClosableIterator<Entry<ByteArray, - List<Versioned<byte[]>>>> entries = + + IClosableIterator<Entry<ByteArray, + List<Versioned<byte[]>>>> entries = store.entries(); try { - SyncMessage bsm = - TProtocolUtil.getTSyncOfferMessage(store.getName(), + SyncMessage bsm = + TProtocolUtil.getTSyncOfferMessage(store.getName(), store.getScope(), store.isPersistent()); int count = 0; while (entries.hasNext()) { if (!rpcService.isConnected(node.getNodeId())) return; - Entry<ByteArray, List<Versioned<byte[]>>> pair = + Entry<ByteArray, List<Versioned<byte[]>>> pair = entries.next(); - KeyedVersions kv = - TProtocolUtil.getTKeyedVersions(pair.getKey(), + KeyedVersions kv = + TProtocolUtil.getTKeyedVersions(pair.getKey(), pair.getValue()); bsm.getSyncOffer().addToVersions(kv); count += 1; @@ -277,9 +280,9 @@ public class SyncManager extends AbstractSyncManager { int rn = random.nextInt(numNodes); antientropy(nodes[rn]); } - + /** - * Write a value synchronized from another node, bypassing some of the + * Write a value synchronized from another node, bypassing some of the * usual logic when a client writes data. If the store is not known, * this will automatically register it * @param storeName the store name @@ -287,11 +290,11 @@ public class SyncManager extends AbstractSyncManager { * @param persist TODO * @param key the key to write * @param values a list of versions for the key to write - * @throws PersistException + * @throws PersistException */ public void writeSyncValue(String storeName, Scope scope, boolean persist, - byte[] key, Iterable<Versioned<byte[]>> values) + byte[] key, Iterable<Versioned<byte[]>> values) throws PersistException { SynchronizingStorageEngine store = storeRegistry.get(storeName); if (store == null) { @@ -311,11 +314,11 @@ public class SyncManager extends AbstractSyncManager { */ public boolean handleSyncOffer(String storeName, byte[] key, - Iterable<VectorClock> versions) + Iterable<VectorClock> versions) throws SyncException { SynchronizingStorageEngine store = storeRegistry.get(storeName); if (store == null) return true; - + List<Versioned<byte[]>> values = store.get(new ByteArray(key)); if (values == null || values.size() == 0) return true; @@ -327,22 +330,22 @@ public class SyncManager extends AbstractSyncManager { return true; } } - + return false; } /** - * Get access to the raw storage engine. This is useful for some + * Get access to the raw storage engine. This is useful for some * on-the-wire communication * @param storeName the store name to get * @return the {@link IStorageEngine} * @throws UnknownStoreException */ - public IStorageEngine<ByteArray, byte[]> getRawStore(String storeName) + public IStorageEngine<ByteArray, byte[]> getRawStore(String storeName) throws UnknownStoreException { return getStoreInternal(storeName); } - + /** * Return the threadpool * @return the {@link IThreadPoolService} @@ -350,7 +353,7 @@ public class SyncManager extends AbstractSyncManager { public IThreadPoolService getThreadPool() { return threadPool; } - + /** * Queue a synchronization of the specified {@link KeyedValues} to all nodes * assocatiated with the storage engine specified @@ -361,21 +364,21 @@ public class SyncManager extends AbstractSyncManager { message="Sync task queue full and not emptying", explanation="The synchronization service is overloaded", recommendation=LogMessageDoc.CHECK_CONTROLLER) - public void queueSyncTask(SynchronizingStorageEngine e, + public void queueSyncTask(SynchronizingStorageEngine e, ByteArray key, Versioned<byte[]> value) { storeRegistry.queueHint(e.getName(), key, value); } - + @Override - public void addListener(String storeName, MappingStoreListener listener) + public void addListener(String storeName, MappingStoreListener listener) throws UnknownStoreException { SynchronizingStorageEngine store = getStoreInternal(storeName); store.addListener(listener); } - + /** * Update the node configuration to add or remove nodes - * @throws FloodlightModuleException + * @throws FloodlightModuleException */ public void updateConfiguration() { if (updateConfigTask != null) @@ -404,7 +407,7 @@ public class SyncManager extends AbstractSyncManager { cursorMap.put(Integer.valueOf(cursorId), cursor); return cursor; } - + /** * Close the given cursor and remove it from the map * @param cursor the cursor to close @@ -413,17 +416,17 @@ public class SyncManager extends AbstractSyncManager { cursor.close(); cursorMap.remove(Integer.valueOf(cursor.getCursorId())); } - + // ******************* // AbstractSyncManager // ******************* @Override - public IStore<ByteArray,byte[]> getStore(String storeName) + public IStore<ByteArray,byte[]> getStore(String storeName) throws UnknownStoreException { return getRawStore(storeName); } - + @Override public short getLocalNodeId() { Node l = clusterConfig.getNode(); @@ -461,7 +464,7 @@ public class SyncManager extends AbstractSyncManager { debugCounter = context.getServiceImpl(IDebugCounterService.class); Map<String, String> config = context.getConfigParams(this); storeRegistry = new StoreRegistry(this, config.get("dbPath")); - + String[] configProviders = {PropertyCCProvider.class.getName(), SyncStoreCCProvider.class.getName(), @@ -493,8 +496,8 @@ public class SyncManager extends AbstractSyncManager { if (manualStoreString != null) { List<String> manualStores = null; try { - manualStores = - (new ObjectMapper()).readValue(manualStoreString, + manualStores = + (new ObjectMapper()).readValue(manualStoreString, new TypeReference<List<String>>() {}); } catch (Exception e) { throw new FloodlightModuleException("Failed to parse sync " + @@ -507,55 +510,60 @@ public class SyncManager extends AbstractSyncManager { } @Override - public void startUp(FloodlightModuleContext context) + public void startUp(FloodlightModuleContext context) throws FloodlightModuleException { if (context != null) { - debugCounter.registerCounter(COUNTER_HINTS, - "Queued sync events processed", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_SENT_VALUES, - "Values synced to remote node", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_RECEIVED_VALUES, - "Values received from remote node", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_PUTS, - "Local puts to store", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_GETS, - "Local gets from store", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_ITERATORS, - "Local iterators created over store", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_ERROR_REMOTE, - "Number of errors sent from remote clients", - CounterType.ALWAYS_COUNT); - debugCounter.registerCounter(COUNTER_ERROR_PROCESSING, - "Number of errors processing messages from remote clients", - CounterType.ALWAYS_COUNT); + try { + COUNTER_HINTS = debugCounter.registerCounter(PACKAGE, " hints", + "Queued sync events processed", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_SENT_VALUES = debugCounter.registerCounter(PACKAGE, "sent-values", + "Values synced to remote node", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_RECEIVED_VALUES = debugCounter.registerCounter(PACKAGE, "received-values", + "Values received from remote node", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_PUTS = debugCounter.registerCounter(PACKAGE, "puts", + "Local puts to store", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_GETS = debugCounter.registerCounter(PACKAGE, "gets", + "Local gets from store", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_ITERATORS = debugCounter.registerCounter(PACKAGE, "iterators", + "Local iterators created over store", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_ERROR_REMOTE = debugCounter.registerCounter(PACKAGE, "error-remote", + "Number of errors sent from remote clients", + CounterType.ALWAYS_COUNT, new Object[] {}); + COUNTER_ERROR_PROCESSING = debugCounter.registerCounter(PACKAGE, "error-processing", + "Number of errors processing messages from remote clients", + CounterType.ALWAYS_COUNT, new Object[] {}); + } catch (MaxCountersRegistered e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } rpcService = new RPCService(this, debugCounter); - - cleanupTask = new SingletonTask(threadPool.getScheduledExecutor(), + + cleanupTask = new SingletonTask(threadPool.getScheduledExecutor(), new CleanupTask()); - cleanupTask.reschedule(CLEANUP_INTERVAL + + cleanupTask.reschedule(CLEANUP_INTERVAL + random.nextInt(30), TimeUnit.SECONDS); - + antientropyTask = new SingletonTask(threadPool.getScheduledExecutor(), new AntientropyTask()); - antientropyTask.reschedule(ANTIENTROPY_INTERVAL + + antientropyTask.reschedule(ANTIENTROPY_INTERVAL + random.nextInt(30), TimeUnit.SECONDS); final ThreadGroup tg = new ThreadGroup("Hint Workers"); tg.setMaxPriority(Thread.NORM_PRIORITY - 2); ThreadFactory f = new ThreadFactory() { AtomicInteger id = new AtomicInteger(); - + @Override public Thread newThread(Runnable runnable) { - return new Thread(tg, runnable, + return new Thread(tg, runnable, "HintWorker-" + id.getAndIncrement()); } }; @@ -599,13 +607,13 @@ public class SyncManager extends AbstractSyncManager { }) protected void doUpdateConfiguration() throws FloodlightModuleException { - + try { ClusterConfig oldConfig = clusterConfig; clusterConfig = clusterConfigProvider.getConfig(); if (clusterConfig.equals(oldConfig)) return; - logger.info("[{}] Updating sync configuration {}", + logger.info("[{}] Updating sync configuration {}", clusterConfig.getNode().getNodeId(), clusterConfig); if (oldConfig.getNode() != null && @@ -642,30 +650,30 @@ public class SyncManager extends AbstractSyncManager { } } - protected SynchronizingStorageEngine getStoreInternal(String storeName) + protected SynchronizingStorageEngine getStoreInternal(String storeName) throws UnknownStoreException { SynchronizingStorageEngine store = storeRegistry.get(storeName); if (store == null) { - throw new UnknownStoreException("Store " + storeName + + throw new UnknownStoreException("Store " + storeName + " has not been registered"); } return store; } - - private void sendSyncOffer(short nodeId, SyncMessage bsm) + + private void sendSyncOffer(short nodeId, SyncMessage bsm) throws InterruptedException { SyncOfferMessage som = bsm.getSyncOffer(); if (!som.isSetVersions()) return; if (logger.isTraceEnabled()) { - logger.trace("[{}->{}] Sending SyncOffer with {} elements", - new Object[]{getLocalNodeId(), nodeId, + logger.trace("[{}->{}] Sending SyncOffer with {} elements", + new Object[]{getLocalNodeId(), nodeId, som.getVersionsSize()}); } som.getHeader().setTransactionId(rpcService.getTransactionId()); rpcService.writeToNode(nodeId, bsm); } - + /** * Periodically perform cleanup * @author readams @@ -685,7 +693,7 @@ public class SyncManager extends AbstractSyncManager { } if (rpcService != null) { - cleanupTask.reschedule(CLEANUP_INTERVAL + + cleanupTask.reschedule(CLEANUP_INTERVAL + random.nextInt(30), TimeUnit.SECONDS); } } @@ -710,13 +718,13 @@ public class SyncManager extends AbstractSyncManager { } if (rpcService != null) { - antientropyTask.reschedule(ANTIENTROPY_INTERVAL + - random.nextInt(30), + antientropyTask.reschedule(ANTIENTROPY_INTERVAL + + random.nextInt(30), TimeUnit.SECONDS); } } } - + /** * Worker task to periodically rescan the configuration * @author readams @@ -735,12 +743,12 @@ public class SyncManager extends AbstractSyncManager { logger.error("Failed to update configuration", e); } if (rpcService != null) { - updateConfigTask.reschedule(CONFIG_RESCAN_INTERVAL, + updateConfigTask.reschedule(CONFIG_RESCAN_INTERVAL, TimeUnit.SECONDS); } } } - + /** * Worker thread that will drain the sync item queue and write the * appropriate messages to the node I/O channels @@ -752,7 +760,7 @@ public class SyncManager extends AbstractSyncManager { recommendation=LogMessageDoc.REPORT_CONTROLLER_BUG) protected class HintWorker implements Runnable { ArrayList<Hint> tasks = new ArrayList<Hint>(50); - protected Map<String, SyncMessage> messages = + protected Map<String, SyncMessage> messages = new LinkedHashMap<String, SyncMessage>(); @Override @@ -763,22 +771,22 @@ public class SyncManager extends AbstractSyncManager { // XXX - todo - handle hints targeted to specific nodes storeRegistry.takeHints(tasks, 50); for (Hint task : tasks) { - debugCounter.updateCounter(COUNTER_HINTS); - SynchronizingStorageEngine store = + debugCounter.updateCounter(COUNTER_HINTS, true); + SynchronizingStorageEngine store = storeRegistry.get(task.getHintKey(). getStoreName()); SyncMessage bsm = getMessage(store); - KeyedValues kv = + KeyedValues kv = TProtocolUtil. - getTKeyedValues(task.getHintKey().getKey(), + getTKeyedValues(task.getHintKey().getKey(), task.getValues()); bsm.getSyncValue().addToValues(kv); } - + Iterable<Node> nodes = getClusterConfig().getNodes(); - short localDomainId = + short localDomainId = getClusterConfig().getNode().getDomainId(); - short localNodeId = + short localNodeId = getClusterConfig().getNode().getNodeId(); for (Node n : nodes) { if (localNodeId == n.getNodeId()) @@ -796,14 +804,13 @@ public class SyncManager extends AbstractSyncManager { svm.getHeader(). setTransactionId(rpcService. getTransactionId()); - debugCounter.updateCounter(COUNTER_SENT_VALUES, + debugCounter.updateCounter(COUNTER_SENT_VALUES, bsm.getSyncValue(). - getValuesSize()); + getValuesSize(), true); rpcService.writeToNode(n.getNodeId(), bsm); } } - debugCounter.flushCounters(); - tasks.clear(); + tasks.clear(); clearMessages(); } catch (Exception e) { @@ -811,16 +818,16 @@ public class SyncManager extends AbstractSyncManager { } } } - + /** * Clear the current list of pending messages */ private void clearMessages() { messages.clear(); } - + /** - * Allocate a partially-initialized {@link SyncMessage} object for + * Allocate a partially-initialized {@link SyncMessage} object for * the given store * @param store the store * @return the {@link SyncMessage} object @@ -829,7 +836,7 @@ public class SyncManager extends AbstractSyncManager { String storeName = store.getName(); SyncMessage bsm = messages.get(storeName); if (bsm == null) { - bsm = TProtocolUtil.getTSyncValueMessage(storeName, + bsm = TProtocolUtil.getTSyncValueMessage(storeName, store.getScope(), store.isPersistent()); messages.put(storeName, bsm); diff --git a/src/main/java/org/sdnplatform/sync/internal/rpc/RPCChannelHandler.java b/src/main/java/org/sdnplatform/sync/internal/rpc/RPCChannelHandler.java index df9c7407c..ef463fd5e 100644 --- a/src/main/java/org/sdnplatform/sync/internal/rpc/RPCChannelHandler.java +++ b/src/main/java/org/sdnplatform/sync/internal/rpc/RPCChannelHandler.java @@ -84,9 +84,8 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { super.messageReceived(ctx, e); - rpcService.debugCounter.flushCounters(); } - + @Override @LogMessageDoc(level="ERROR", message="[{id}->{id}] Attempted connection from unrecognized " + @@ -98,7 +97,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { "in your network.") protected void handleHello(HelloMessage hello, Channel channel) { if (!hello.isSetNodeId()) { - // this is a client connection. Don't set this up as a node + // this is a client connection. Don't set this up as a node // connection isClientConnection = true; return; @@ -107,7 +106,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { if (remoteNode == null) { logger.error("[{}->{}] Attempted connection from unrecognized " + "floodlight node {}; disconnecting", - new Object[]{getLocalNodeIdString(), + new Object[]{getLocalNodeIdString(), getRemoteNodeIdString(), hello.getNodeId()}); channel.close(); @@ -121,7 +120,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { srm.setHeader(header); SyncMessage bsm = new SyncMessage(MessageType.FULL_SYNC_REQUEST); channel.write(bsm); - + // XXX - TODO - if last connection was longer ago than the tombstone // timeout, then we need to do a complete flush and reload of our // state. This is complex though since this applies across entire @@ -133,47 +132,47 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { } @Override - protected void handleGetRequest(GetRequestMessage request, + protected void handleGetRequest(GetRequestMessage request, Channel channel) { String storeName = request.getStoreName(); try { - IStorageEngine<ByteArray, byte[]> store = + IStorageEngine<ByteArray, byte[]> store = syncManager.getRawStore(storeName); - + GetResponseMessage m = new GetResponseMessage(); AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); m.setHeader(header); - - List<Versioned<byte[]>> values = + + List<Versioned<byte[]>> values = store.get(new ByteArray(request.getKey())); for (Versioned<byte[]> value : values) { m.addToValues(TProtocolUtil.getTVersionedValue(value)); } - + SyncMessage bsm = new SyncMessage(MessageType.GET_RESPONSE); bsm.setGetResponse(m); channel.write(bsm); } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.GET_REQUEST)); } } - + @Override - protected void handlePutRequest(PutRequestMessage request, + protected void handlePutRequest(PutRequestMessage request, Channel channel) { String storeName = request.getStoreName(); try { - IStorageEngine<ByteArray, byte[]> store = + IStorageEngine<ByteArray, byte[]> store = syncManager.getRawStore(storeName); - + ByteArray key = new ByteArray(request.getKey()); Versioned<byte[]> value = null; if (request.isSetVersionedValue()) { value = TProtocolUtil. getVersionedValued(request.getVersionedValue()); - value.increment(syncManager.getLocalNodeId(), + value.increment(syncManager.getLocalNodeId(), System.currentTimeMillis()); } else if (request.isSetValue()) { byte[] rvalue = request.getValue(); @@ -182,7 +181,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { for (IVersion v : versions) { newclock = newclock.merge((VectorClock)v); } - newclock = newclock.incremented(syncManager.getLocalNodeId(), + newclock = newclock.incremented(syncManager.getLocalNodeId(), System.currentTimeMillis()); value = Versioned.value(rvalue, newclock); } else { @@ -195,22 +194,22 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); m.setHeader(header); - + SyncMessage bsm = new SyncMessage(MessageType.PUT_RESPONSE); bsm.setPutResponse(m); channel.write(bsm); } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.PUT_REQUEST)); } } - + @Override - protected void handleDeleteRequest(DeleteRequestMessage request, + protected void handleDeleteRequest(DeleteRequestMessage request, Channel channel) { try { String storeName = request.getStoreName(); - IStorageEngine<ByteArray, byte[]> store = + IStorageEngine<ByteArray, byte[]> store = syncManager.getRawStore(storeName); ByteArray key = new ByteArray(request.getKey()); VectorClock newclock; @@ -223,23 +222,23 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { newclock = newclock.merge((VectorClock)v); } } - newclock = - newclock.incremented(rpcService.syncManager.getLocalNodeId(), + newclock = + newclock.incremented(rpcService.syncManager.getLocalNodeId(), System.currentTimeMillis()); Versioned<byte[]> value = Versioned.value(null, newclock); store.put(key, value); - + DeleteResponseMessage m = new DeleteResponseMessage(); AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); m.setHeader(header); - SyncMessage bsm = + SyncMessage bsm = new SyncMessage(MessageType.DELETE_RESPONSE); bsm.setDeleteResponse(m); channel.write(bsm); } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.DELETE_REQUEST)); } } @@ -248,13 +247,13 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { protected void handleSyncValue(SyncValueMessage request, Channel channel) { if (request.isSetResponseTo()) - rpcService.messageAcked(MessageType.SYNC_REQUEST, + rpcService.messageAcked(MessageType.SYNC_REQUEST, getRemoteNodeId()); try { if (logger.isTraceEnabled()) { - logger.trace("[{}->{}] Got syncvalue {}", - new Object[]{getLocalNodeIdString(), - getRemoteNodeIdString(), + logger.trace("[{}->{}] Got syncvalue {}", + new Object[]{getLocalNodeIdString(), + getRemoteNodeIdString(), request}); } @@ -264,7 +263,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { Iterable<Versioned<byte[]>> vs = new TVersionedValueIterable(tvvi); syncManager.writeSyncValue(request.getStore().getStoreName(), scope, - request.getStore().isPersist(), + request.getStore().isPersist(), kv.getKey(), vs); } @@ -273,76 +272,77 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); m.setHeader(header); - SyncMessage bsm = + SyncMessage bsm = new SyncMessage(MessageType.SYNC_VALUE_RESPONSE); bsm.setSyncValueResponse(m); - - updateCounter(SyncManager.COUNTER_RECEIVED_VALUES, + + updateCounter(SyncManager.COUNTER_RECEIVED_VALUES, request.getValuesSize()); channel.write(bsm); } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.SYNC_VALUE)); } } - - protected void handleSyncValueResponse(SyncValueResponseMessage message, + + @Override + protected void handleSyncValueResponse(SyncValueResponseMessage message, Channel channel) { rpcService.messageAcked(MessageType.SYNC_VALUE, getRemoteNodeId()); } @Override - protected void handleSyncOffer(SyncOfferMessage request, + protected void handleSyncOffer(SyncOfferMessage request, Channel channel) { try { String storeName = request.getStore().getStoreName(); - + SyncRequestMessage srm = new SyncRequestMessage(); AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); srm.setHeader(header); srm.setStore(request.getStore()); - + for (KeyedVersions kv : request.getVersions()) { - Iterable<org.sdnplatform.sync.thrift.VectorClock> tvci = + Iterable<org.sdnplatform.sync.thrift.VectorClock> tvci = kv.getVersions(); Iterable<VectorClock> vci = new TVersionIterable(tvci); - - boolean wantKey = syncManager.handleSyncOffer(storeName, + + boolean wantKey = syncManager.handleSyncOffer(storeName, kv.getKey(), vci); if (wantKey) srm.addToKeys(kv.bufferForKey()); } - - SyncMessage bsm = + + SyncMessage bsm = new SyncMessage(MessageType.SYNC_REQUEST); bsm.setSyncRequest(srm); if (logger.isTraceEnabled()) { - logger.trace("[{}->{}] Sending SyncRequest with {} elements", - new Object[]{getLocalNodeIdString(), - getRemoteNodeIdString(), + logger.trace("[{}->{}] Sending SyncRequest with {} elements", + new Object[]{getLocalNodeIdString(), + getRemoteNodeIdString(), srm.getKeysSize()}); } channel.write(bsm); - + } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.SYNC_OFFER)); } } @Override - protected void handleSyncRequest(SyncRequestMessage request, + protected void handleSyncRequest(SyncRequestMessage request, Channel channel) { rpcService.messageAcked(MessageType.SYNC_OFFER, getRemoteNodeId()); if (!request.isSetKeys()) return; String storeName = request.getStore().getStoreName(); try { - IStorageEngine<ByteArray, byte[]> store = + IStorageEngine<ByteArray, byte[]> store = syncManager.getRawStore(storeName); - SyncMessage bsm = + SyncMessage bsm = TProtocolUtil.getTSyncValueMessage(request.getStore()); SyncValueMessage svm = bsm.getSyncValue(); svm.setResponseTo(request.getHeader().getTransactionId()); @@ -350,22 +350,22 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { for (ByteBuffer key : request.getKeys()) { ByteArray keyArray = new ByteArray(key.array()); - List<Versioned<byte[]>> values = + List<Versioned<byte[]>> values = store.get(keyArray); if (values == null || values.size() == 0) continue; - KeyedValues kv = + KeyedValues kv = TProtocolUtil.getTKeyedValues(keyArray, values); svm.addToValues(kv); } - + if (svm.isSetValues()) { - updateCounter(SyncManager.COUNTER_SENT_VALUES, + updateCounter(SyncManager.COUNTER_SENT_VALUES, svm.getValuesSize()); rpcService.syncQueue.add(new NodeMessage(getRemoteNodeId(), bsm)); } } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.SYNC_REQUEST)); } } @@ -376,6 +376,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { startAntientropy(); } + @Override protected void handleCursorRequest(CursorRequestMessage request, Channel channel) { try { @@ -401,14 +402,14 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { int i = 0; while (i < 50 && c.hasNext()) { Entry<ByteArray, List<Versioned<byte[]>>> e = c.next(); - - m.addToValues(TProtocolUtil.getTKeyedValues(e.getKey(), + + m.addToValues(TProtocolUtil.getTKeyedValues(e.getKey(), e.getValue())); i += 1; } } - SyncMessage bsm = + SyncMessage bsm = new SyncMessage(MessageType.CURSOR_RESPONSE); bsm.setCursorResponse(m); channel.write(bsm); @@ -424,7 +425,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { try { Scope scope = TProtocolUtil.getScope(request.store.getScope()); if (request.store.isPersist()) - syncManager.registerPersistentStore(request.store.storeName, + syncManager.registerPersistentStore(request.store.storeName, scope); else syncManager.registerStore(request.store.storeName, scope); @@ -432,12 +433,12 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); m.setHeader(header); - SyncMessage bsm = + SyncMessage bsm = new SyncMessage(MessageType.REGISTER_RESPONSE); bsm.setRegisterResponse(m); channel.write(bsm); } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.REGISTER_REQUEST)); } } @@ -446,16 +447,16 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { protected void handleClusterJoinRequest(ClusterJoinRequestMessage request, Channel channel) { try { - // We can get this message in two circumstances. Either this is + // We can get this message in two circumstances. Either this is // a totally new node, or this is an existing node that is changing // its port or IP address. We can tell the difference because the // node ID and domain ID will already be set for an existing node - + ClusterJoinResponseMessage cjrm = new ClusterJoinResponseMessage(); AsyncMessageHeader header = new AsyncMessageHeader(); header.setTransactionId(request.getHeader().getTransactionId()); cjrm.setHeader(header); - + org.sdnplatform.sync.thrift.Node tnode = request.getNode(); if (!tnode.isSetNodeId()) { // allocate a random node ID that's not currently in use @@ -483,13 +484,13 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { // new domain ID into the system node store tnode.setDomainId(tnode.getNodeId()); } - IStoreClient<Short, Node> nodeStoreClient = + IStoreClient<Short, Node> nodeStoreClient = syncManager.getStoreClient(SyncStoreCCProvider. SYSTEM_NODE_STORE, Short.class, Node.class); while (true) { try { - Versioned<Node> node = + Versioned<Node> node = nodeStoreClient.get(tnode.getNodeId()); node.setValue(new Node(tnode.getHostname(), tnode.getPort(), @@ -499,30 +500,30 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { break; } catch (ObsoleteVersionException e) { } } - - IStorageEngine<ByteArray, byte[]> store = + + IStorageEngine<ByteArray, byte[]> store = syncManager.getRawStore(SyncStoreCCProvider. SYSTEM_NODE_STORE); - IClosableIterator<Entry<ByteArray, + IClosableIterator<Entry<ByteArray, List<Versioned<byte[]>>>> entries = store.entries(); try { while (entries.hasNext()) { - Entry<ByteArray, List<Versioned<byte[]>>> entry = + Entry<ByteArray, List<Versioned<byte[]>>> entry = entries.next(); - KeyedValues kv = - TProtocolUtil.getTKeyedValues(entry.getKey(), + KeyedValues kv = + TProtocolUtil.getTKeyedValues(entry.getKey(), entry.getValue()); cjrm.addToNodeStore(kv); } } finally { entries.close(); } - SyncMessage bsm = + SyncMessage bsm = new SyncMessage(MessageType.CLUSTER_JOIN_RESPONSE); bsm.setClusterJoinResponse(cjrm); channel.write(bsm); } catch (Exception e) { - channel.write(getError(request.getHeader().getTransactionId(), e, + channel.write(getError(request.getHeader().getTransactionId(), e, MessageType.CLUSTER_JOIN_REQUEST)); } } @@ -542,10 +543,10 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { protected Short getLocalNodeId() { return syncManager.getLocalNodeId(); } - + @Override protected Short getRemoteNodeId() { - if (remoteNode != null) + if (remoteNode != null) return remoteNode.getNodeId(); return null; } @@ -554,7 +555,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { protected String getLocalNodeIdString() { return ""+getLocalNodeId(); } - + @Override protected String getRemoteNodeIdString() { return ""+getRemoteNodeId(); @@ -564,7 +565,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { protected int getTransactionId() { return rpcService.getTransactionId(); } - + @Override protected AuthScheme getAuthScheme() { return syncManager.getClusterConfig().getAuthScheme(); @@ -577,12 +578,13 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { try { return CryptoUtil.getSharedSecret(path, pass); } catch (Exception e) { - throw new AuthException("Could not read challenge/response " + + throw new AuthException("Could not read challenge/response " + "shared secret from key store " + path, e); } } - - protected SyncMessage getError(int transactionId, Exception error, + + @Override + protected SyncMessage getError(int transactionId, Exception error, MessageType type) { updateCounter(SyncManager.COUNTER_ERROR_PROCESSING, 1); return super.getError(transactionId, error, type); @@ -592,10 +594,10 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { // Utility functions // ***************** - protected void updateCounter(String counter, int incr) { - rpcService.debugCounter.updateCounter(counter, incr); + protected void updateCounter(int counter, int incr) { + rpcService.debugCounter.updateCounter(counter, incr, true); } - + protected void startAntientropy() { // Run antientropy in a background task so we don't use up an I/O // thread. Note that this task will result in lots of traffic @@ -621,7 +623,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler { @Override public Iterator<VectorClock> iterator() { - final Iterator<org.sdnplatform.sync.thrift.VectorClock> tcs = + final Iterator<org.sdnplatform.sync.thrift.VectorClock> tcs = tcvi.iterator(); return new Iterator<VectorClock>() { diff --git a/src/main/java/org/sdnplatform/sync/internal/store/ListenerStorageEngine.java b/src/main/java/org/sdnplatform/sync/internal/store/ListenerStorageEngine.java index 38f77e4d3..f64591322 100644 --- a/src/main/java/org/sdnplatform/sync/internal/store/ListenerStorageEngine.java +++ b/src/main/java/org/sdnplatform/sync/internal/store/ListenerStorageEngine.java @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory; * @author readams */ @LogMessageCategory("State Synchronization") -public class ListenerStorageEngine +public class ListenerStorageEngine implements IStorageEngine<ByteArray, byte[]> { protected static Logger logger = LoggerFactory.getLogger(ListenerStorageEngine.class); @@ -34,7 +34,7 @@ public class ListenerStorageEngine /** * Listeners for this store */ - protected List<MappingStoreListener> listeners = + protected List<MappingStoreListener> listeners = new ArrayList<MappingStoreListener>(); /** @@ -46,7 +46,7 @@ public class ListenerStorageEngine * Debug counter service */ protected IDebugCounterService debugCounter; - + /** * Allocate new {@link ListenerStorageEngine} * @param localStorage the delegate store @@ -130,7 +130,7 @@ public class ListenerStorageEngine public void setTombstoneInterval(int interval) { localStorage.setTombstoneInterval(interval); } - + // ********************* // ListenerStorageEngine // ********************* @@ -145,7 +145,7 @@ public class ListenerStorageEngine @LogMessageDoc(level="ERROR", message="An error occurred in a sync listener", - explanation="An unexpected error occured in a handler for " + + explanation="An unexpected error occured in a handler for " + "an update to shared state.") protected void notifyListeners(Iterator<ByteArray> keys, UpdateType type) { for (MappingStoreListener msl : listeners) { @@ -157,9 +157,9 @@ public class ListenerStorageEngine } } - protected void updateCounter(String counterName) { + protected void updateCounter(int counterName) { if (debugCounter != null) { - debugCounter.updateCounter(counterName); + debugCounter.updateCounter(counterName, true); } - } + } } diff --git a/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyBelowTest.java b/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyBelowTest.java new file mode 100644 index 000000000..e44cfe590 --- /dev/null +++ b/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyBelowTest.java @@ -0,0 +1,169 @@ +package net.floodlightcontroller.debugcounter; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.floodlightcontroller.debugcounter.DebugCounter.CounterIndexStore; +import net.floodlightcontroller.debugcounter.DebugCounter.RetCtrInfo; +import net.floodlightcontroller.test.FloodlightTestCase; + +public class CounterHierarchyBelowTest extends FloodlightTestCase { + DebugCounter dc; + protected static Logger log = LoggerFactory.getLogger(CounterHierarchyBelowTest.class); + + ConcurrentHashMap<String, ConcurrentHashMap<String, CounterIndexStore>> mctr; + ArrayList<Integer> exp; + + + @Override + @Before + public void setUp() throws Exception { + dc = new DebugCounter(); + mctr = dc.moduleCounters; + + mctr.put("switch", new ConcurrentHashMap<String, CounterIndexStore>()); + RetCtrInfo rci = dc.getCounterId("switch", "01"); + dc.addToModuleCounterHierarchy("switch", 4, rci); + rci = dc.getCounterId("switch", "01/pktin"); + dc.addToModuleCounterHierarchy("switch", 42, rci); + rci = dc.getCounterId("switch", "01/pktout"); + dc.addToModuleCounterHierarchy("switch", 47, rci); + rci = dc.getCounterId("switch", "01/pktin/drops"); + dc.addToModuleCounterHierarchy("switch", 427, rci); + rci = dc.getCounterId("switch", "01/pktin/err"); + dc.addToModuleCounterHierarchy("switch", 428, rci); + + rci = dc.getCounterId("switch", "02"); + dc.addToModuleCounterHierarchy("switch", 8, rci); + + mctr.put("linkd", new ConcurrentHashMap<String, CounterIndexStore>()); + rci = dc.getCounterId("linkd", "tunnel"); + dc.addToModuleCounterHierarchy("linkd", 2, rci); + mctr.put("sinkd", new ConcurrentHashMap<String, CounterIndexStore>()); + rci = dc.getCounterId("sinkd", "tunnel"); + dc.addToModuleCounterHierarchy("sinkd", 5, rci); + + exp = new ArrayList<Integer>(); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 8, 2, 5); + exp.addAll(temp); + } + + private void isEqual(ArrayList<Integer> a, ArrayList<Integer> b) { + if (a.size() != b.size() || !b.containsAll(a)) assertTrue(false); + } + + + @Test + public void testHierarchyAll() { + RetCtrInfo rci = dc.new RetCtrInfo(); + ArrayList<Integer> retval = new ArrayList<Integer>(); + + for (String moduleName : mctr.keySet()) { + ArrayList<Integer> ids = dc.getHierarchyBelow(moduleName, rci); + retval.addAll(ids); + } + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + + @Test + public void testHierarchy0() { + RetCtrInfo rci = dc.getCounterId("switch", ""); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(2, 5); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy0a() { + RetCtrInfo rci = dc.getCounterId("linkd", ""); + ArrayList<Integer> retval = dc.getHierarchyBelow("linkd", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy1() { + RetCtrInfo rci = dc.getCounterId("switch", "01"); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(4, 8, 2, 5); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy1a() { + RetCtrInfo rci = dc.getCounterId("switch", "02"); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 2, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy1b() { + RetCtrInfo rci = dc.getCounterId("sinkd", "tunnel"); + ArrayList<Integer> retval = dc.getHierarchyBelow("sinkd", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 2, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy2() { + RetCtrInfo rci = dc.getCounterId("switch", "01/pktin"); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 2, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy2a() { + RetCtrInfo rci = dc.getCounterId("switch", "01/pktout"); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 2, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy2b() { + RetCtrInfo rci = dc.getCounterId("switch", "02/pktin"); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 2, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + @Test + public void testHierarchy3() { + RetCtrInfo rci = dc.getCounterId("switch", "01/pktin/drops"); + ArrayList<Integer> retval = dc.getHierarchyBelow("switch", rci); + List<Integer> temp = Arrays.asList(4, 42, 47, 427, 428, 2, 5, 8); + exp.removeAll(temp); + log.info("got==> {}, exp=> {}", retval, exp); + isEqual(retval, exp); + } + + + +} diff --git a/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyGetTest.java b/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyGetTest.java new file mode 100644 index 000000000..f82972f14 --- /dev/null +++ b/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyGetTest.java @@ -0,0 +1,278 @@ +package net.floodlightcontroller.debugcounter; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import net.floodlightcontroller.debugcounter.DebugCounter.CounterIndexStore; +import net.floodlightcontroller.debugcounter.DebugCounter.RetCtrInfo; +import net.floodlightcontroller.test.FloodlightTestCase; + +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CounterHierarchyGetTest extends FloodlightTestCase { + DebugCounter dc; + protected static Logger log = LoggerFactory.getLogger(CounterHierarchyGetTest.class); + + ConcurrentHashMap<String, ConcurrentHashMap<String, CounterIndexStore>> mctr; + RetCtrInfo exp; + + @Override + @Before + public void setUp() throws Exception { + dc = new DebugCounter(); + mctr = dc.moduleCounters; + mctr.put("linkd", new ConcurrentHashMap<String, CounterIndexStore>()); + mctr.get("linkd").put("linkevent", dc.new CounterIndexStore(24, null)); + mctr.put("switch", new ConcurrentHashMap<String, CounterIndexStore>()); + exp = dc.new RetCtrInfo(); + } + + @Test + public void testBasicCounterGet() { + RetCtrInfo rci = dc.getCounterId("linkd", "linkevent"); + //exp.levels = new String[IDebugCounterService.MAX_HIERARCHY]; + exp.levels = "linkevent".split("/"); + exp.allLevelsFound = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 24; + exp.levels[0] = "linkevent"; + assertEquals(exp, rci); + } + + @Test + public void testHierarchicalCounterGet1() { + Map<String, CounterIndexStore> x = mctr.get("switch"); + x.put("00:00:00:00:00:00:00:01", + dc.new CounterIndexStore(10, null)); + + // 1-level counter exists + String counterName = "00:00:00:00:00:00:00:01"; + RetCtrInfo rci1 = dc.getCounterId("switch",counterName); + exp.levels = counterName.split("/"); + exp.allLevelsFound = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 10; + exp.levels[0] = counterName; + assertEquals(exp, rci1); + + // 1-level counter does not exist + counterName = "00:00:00:00:00:00:00:02"; + RetCtrInfo rci2 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.foundUptoLevel = 0; + exp.hierarchical = false; + exp.levels = counterName.split("/"); + exp.ctrIds[0] = -1; + printRCI("got==> ", rci2); + printRCI("exp==> ", exp); + assertEquals(exp, rci2); + + // 2-level hierarchical counter does not exist + counterName = "00:00:00:00:00:00:00:01/pktin"; + RetCtrInfo rci3 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 10; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci3); + printRCI("exp==> ", exp); + assertEquals(exp, rci3); + + // 3-level hierarchical counter does not exist + counterName = "00:00:00:00:00:00:00:01/pktin/drops"; + RetCtrInfo rci4 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 10; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci4); + printRCI("exp==> ", exp); + assertEquals(exp, rci4); + + // 4-level hierarchical counter does not exist + counterName = "00:00:00:00:00:00:00:01/pktin/drops/extra"; + RetCtrInfo rci5 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 10; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci5); + printRCI("exp==> ", exp); + assertEquals(exp, rci5); + + } + + @Test + public void testHierarchicalCounterGet2() { + Map<String, CounterIndexStore> x = mctr.get("switch"); + // single level counter + x.put("00:00:00:00:00:00:00:01", + dc.new CounterIndexStore(10, null)); + // two level counter + x.put("00:00:00:00:00:00:00:03", + dc.new CounterIndexStore(30, new ConcurrentHashMap<String,CounterIndexStore>())); + x.get("00:00:00:00:00:00:00:03").nextLevel.put("pktin", + dc.new CounterIndexStore(333, null)); + + // 1-level counter exists + String counterName = "00:00:00:00:00:00:00:01"; + RetCtrInfo rci1 = dc.getCounterId("switch",counterName); + exp.levels = counterName.split("/"); + exp.allLevelsFound = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 10; + exp.levels[0] = counterName; + assertEquals(exp, rci1); + + // 1-level counter does not exist + counterName = "00:00:00:00:00:00:00:02"; + RetCtrInfo rci2 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = false; + exp.levels = counterName.split("/"); + exp.foundUptoLevel = 0; + exp.ctrIds[0] = -1; + printRCI("got==> ", rci2); + printRCI("exp==> ", exp); + assertEquals(exp, rci2); + + // 2-level hierarchical counter does not exist + counterName = "00:00:00:00:00:00:00:01/pktin"; + RetCtrInfo rci3 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.ctrIds[0] = 10; + exp.foundUptoLevel = 1; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci3); + printRCI("exp==> ", exp); + assertEquals(exp, rci3); + + // 2-level hierarchical counter DOES exist + counterName = "00:00:00:00:00:00:00:03/pktin"; + RetCtrInfo rci3x = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.hierarchical = true; + exp.foundUptoLevel = 2; + exp.ctrIds[0] = 30; + exp.ctrIds[1] = 333; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci3x); + printRCI("exp==> ", exp); + assertEquals(exp, rci3x); + + // 2-level hierarchical counter does NOT exist + counterName = "00:00:00:00:00:00:00:03/pktout"; + RetCtrInfo rci3y = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 30; + exp.ctrIds[1] = -1; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci3y); + printRCI("exp==> ", exp); + assertEquals(exp, rci3y); + + // 3-level hierarchical counter does not exist + counterName = "00:00:00:00:00:00:00:03/pktin/drops"; + RetCtrInfo rci4 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.foundUptoLevel = 2; + exp.ctrIds[0] = 30; + exp.ctrIds[1] = 333; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci4); + printRCI("exp==> ", exp); + assertEquals(exp, rci4); + + } + + @Test + public void testHierarchicalCounterGet3() { + Map<String, CounterIndexStore> x = mctr.get("switch"); + // single level counter + x.put("00:00:00:00:00:00:00:01", + dc.new CounterIndexStore(10, null)); + // two level counter + x.put("00:00:00:00:00:00:00:03", + dc.new CounterIndexStore(30, new ConcurrentHashMap<String,CounterIndexStore>())); + x.get("00:00:00:00:00:00:00:03").nextLevel.put("pktin", + dc.new CounterIndexStore(333, null)); + // three level counter + x.put("00:00:00:00:00:00:00:05", + dc.new CounterIndexStore(50, new ConcurrentHashMap<String,CounterIndexStore>())); + x.get("00:00:00:00:00:00:00:05") + .nextLevel.put("pktin", + dc.new CounterIndexStore(555, new ConcurrentHashMap<String,CounterIndexStore>())); + x.get("00:00:00:00:00:00:00:05").nextLevel.get("pktin").nextLevel. + put("drops", dc.new CounterIndexStore(1056, null)); + + // 1-level counter exists + String counterName = "00:00:00:00:00:00:00:01"; + RetCtrInfo rci1 = dc.getCounterId("switch",counterName); + exp.levels = counterName.split("/"); + exp.allLevelsFound = true; + exp.foundUptoLevel = 1; + exp.ctrIds[0] = 10; + exp.levels[0] = counterName; + assertEquals(exp, rci1); + + // 2-level hierarchical counter DOES exist + counterName = "00:00:00:00:00:00:00:03/pktin"; + RetCtrInfo rci3x = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.hierarchical = true; + exp.foundUptoLevel = 2; + exp.ctrIds[0] = 30; + exp.ctrIds[1] = 333; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci3x); + printRCI("exp==> ", exp); + assertEquals(exp, rci3x); + + // 3-level hierarchical counter DOES exist + counterName = "00:00:00:00:00:00:00:05/pktin/drops"; + RetCtrInfo rci4 = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.hierarchical = true; + exp.foundUptoLevel = 3; + exp.ctrIds[0] = 50; + exp.ctrIds[1] = 555; + exp.ctrIds[2] = 1056; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci4); + printRCI("exp==> ", exp); + assertEquals(exp, rci4); + + // querying only 2 levels of a 3 level counter + counterName = "00:00:00:00:00:00:00:05/pktin"; + RetCtrInfo rci4x = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.hierarchical = true; + exp.foundUptoLevel = 2; + exp.ctrIds[0] = 50; + exp.ctrIds[1] = 555; + exp.ctrIds[2] = -1; + exp.levels = counterName.split("/"); + printRCI("got==> ", rci4x); + printRCI("exp==> ", exp); + assertEquals(exp, rci4x); + } + + + public void printRCI(String hdr, RetCtrInfo rci) { + log.info(hdr+"found={}, hcy={}, foundUL= {}, idsFound={}, incomingLevels={}", + new Object[] {rci.allLevelsFound, rci.hierarchical, + rci.foundUptoLevel, + rci.ctrIds, rci.levels}); + } + +} diff --git a/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyPutTest.java b/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyPutTest.java new file mode 100644 index 000000000..d039f8f8c --- /dev/null +++ b/src/test/java/net/floodlightcontroller/debugcounter/CounterHierarchyPutTest.java @@ -0,0 +1,121 @@ +package net.floodlightcontroller.debugcounter; + +import java.util.concurrent.ConcurrentHashMap; + +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.floodlightcontroller.debugcounter.DebugCounter.CounterIndexStore; +import net.floodlightcontroller.debugcounter.DebugCounter.RetCtrInfo; +import net.floodlightcontroller.test.FloodlightTestCase; + +public class CounterHierarchyPutTest extends FloodlightTestCase { + DebugCounter dc; + protected static Logger log = LoggerFactory.getLogger(CounterHierarchyPutTest.class); + + ConcurrentHashMap<String, ConcurrentHashMap<String, CounterIndexStore>> mctr; + RetCtrInfo exp; + + @Override + @Before + public void setUp() throws Exception { + dc = new DebugCounter(); + mctr = dc.moduleCounters; + mctr.put("linkd", new ConcurrentHashMap<String, CounterIndexStore>()); + mctr.get("linkd").put("linkevent", dc.new CounterIndexStore(24, null)); + mctr.put("switch", new ConcurrentHashMap<String, CounterIndexStore>()); + exp = dc.new RetCtrInfo(); + } + + @Test + public void testHierarchicalPut() { + String counterName = "100hp"; + RetCtrInfo rci = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = false; + exp.levels = counterName.split("/"); + printRCI("got ==>", rci); + printRCI("exp ==>", exp); + assertEquals(rci, exp); + // add and then check for first level of hierarchy + dc.addToModuleCounterHierarchy("switch", 45, rci); + rci = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.foundUptoLevel = 1; + exp.hierarchical = false; + exp.ctrIds[0] = 45; + exp.levels = counterName.split("/"); + printRCI("got ==>", rci); + printRCI("exp ==>", exp); + assertEquals(rci, exp); + + counterName = "100hp/pktin"; + rci = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.levels = counterName.split("/"); + exp.ctrIds[0] = 45; + printRCI("got ==>", rci); + printRCI("exp ==>", exp); + assertEquals(rci, exp); + dc.printAllCounters(); + // add and then check for 2nd level of hierarchy + dc.addToModuleCounterHierarchy("switch", 77, rci); + rci = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.foundUptoLevel = 2; + exp.hierarchical = true; + exp.ctrIds[0] = 45; + exp.ctrIds[1] = 77; + exp.levels = counterName.split("/"); + printRCI("got ==>", rci); + printRCI("exp ==>", exp); + dc.printAllCounters(); + assertEquals(rci, exp); + + counterName = "100hp/pktin/drops"; + rci = dc.getCounterId("switch", counterName); + exp.allLevelsFound = false; + exp.hierarchical = true; + exp.levels = counterName.split("/"); + exp.ctrIds[0] = 45; + exp.ctrIds[1] = 77; + exp.foundUptoLevel = 2; + printRCI("got ==>", rci); + printRCI("exp ==>", exp); + assertEquals(rci, exp); + dc.printAllCounters(); + // add and then check for 3rd level of hierarchy + dc.addToModuleCounterHierarchy("switch", 132, rci); + rci = dc.getCounterId("switch", counterName); + exp.allLevelsFound = true; + exp.foundUptoLevel = 3; + exp.hierarchical = true; + exp.ctrIds[0] = 45; + exp.ctrIds[1] = 77; + exp.ctrIds[2] = 132; + exp.levels = counterName.split("/"); + printRCI("got ==>", rci); + printRCI("exp ==>", exp); + dc.printAllCounters(); + assertEquals(rci, exp); + + } + + @Test + public void testOtherTest() { + Integer[] test = new Integer[2000]; + log.info("it is: {}", test[56]); + } + + private void printRCI(String hdr, RetCtrInfo rci) { + log.info(hdr+"found={}, hcy={}, foundUL= {}, idsFound={}, incomingLevels={}", + new Object[] {rci.allLevelsFound, rci.hierarchical, + rci.foundUptoLevel, + rci.ctrIds, rci.levels}); + + } + +} -- GitLab