From 22c60b611be29d74be8757d1e1c7ca02c357e480 Mon Sep 17 00:00:00 2001 From: Shudong Zhou <shudongzhou@gmail.com> Date: Sat, 17 Nov 2012 11:27:27 -0800 Subject: [PATCH] Fixes to HA role request handling (Gregor's comments) --- .../core/annotations/LogMessageDoc.java | 4 + .../core/internal/Controller.java | 77 +------ .../core/internal/RoleChanger.java | 214 +++++++++++------- .../core/internal/ControllerTest.java | 21 +- .../core/internal/RoleChangerTest.java | 38 +++- 5 files changed, 169 insertions(+), 185 deletions(-) diff --git a/src/main/java/net/floodlightcontroller/core/annotations/LogMessageDoc.java b/src/main/java/net/floodlightcontroller/core/annotations/LogMessageDoc.java index c703687a1..08b48dc84 100644 --- a/src/main/java/net/floodlightcontroller/core/annotations/LogMessageDoc.java +++ b/src/main/java/net/floodlightcontroller/core/annotations/LogMessageDoc.java @@ -35,6 +35,10 @@ public @interface LogMessageDoc { public static final String CHECK_SWITCH = "Check the health of the indicated switch. " + "Test and troubleshoot IP connectivity."; + public static final String HA_CHECK_SWITCH = + "Check the health of the indicated switch. If the problem " + + "persists or occurs repeatedly, it likely indicates a defect " + + "in the switch HA implementation."; public static final String CHECK_CONTROLLER = "Verify controller system health, CPU usage, and memory. " + "Rebooting the controller node may help if the controller " + diff --git a/src/main/java/net/floodlightcontroller/core/internal/Controller.java b/src/main/java/net/floodlightcontroller/core/internal/Controller.java index dda0432dd..b7b048874 100644 --- a/src/main/java/net/floodlightcontroller/core/internal/Controller.java +++ b/src/main/java/net/floodlightcontroller/core/internal/Controller.java @@ -449,6 +449,7 @@ public class Controller implements IFloodlightProviderService, } synchronized(roleChanger) { connectedSwitches.remove(sw); + roleChanger.removePendingRequests(sw); } sw.setConnected(false); } @@ -917,9 +918,6 @@ public class Controller implements IFloodlightProviderService, sendFeatureReplyConfiguration(); state.featuresReply = (OFFeaturesReply) m; state.hsState = HandshakeState.FEATURES_REPLY; - // uncomment to enable "dumb" switches like cbench - // state.hsState = HandshakeState.READY; - // addSwitch(sw); } else { // return results to rest api caller sw.setFeaturesReply((OFFeaturesReply) m); @@ -956,81 +954,14 @@ public class Controller implements IFloodlightProviderService, // This will probable involve rewriting the way we handle // request/reply style messages. OFError error = (OFError) m; - boolean shouldLogError = true; - // TODO: should we check that firstRoleReplyReceived is false, - // i.e., check only whether the first request fails? if (roleChanger.checkFirstPendingRoleRequestXid( sw, error.getXid())) { - boolean isBadVendorError = - (error.getErrorType() == OFError.OFErrorType. - OFPET_BAD_REQUEST.getValue()); - // We expect to receive a bad vendor error when - // we're connected to a switch that doesn't support - // the Nicira vendor extensions (i.e. not OVS or - // derived from OVS). By protocol, it should also be - // BAD_VENDOR, but too many switch implementations - // get it wrong and we can already check the xid() - // so we can ignore the type with confidence that this - // is not a spurious error - shouldLogError = !isBadVendorError; - if (isBadVendorError) { - if (sw.getHARole() != null) { - log.warn("Received ERROR from sw {} that " - +"indicates roles are not supported " - +"but we have received a valid " - +"role reply earlier", sw); - } - - roleChanger.deliverRoleRequestNotSupported(sw, error.getXid()); - synchronized(roleChanger) { - if (Controller.this.role==Role.SLAVE) { - // the switch doesn't understand role request - // messages and the current controller role is - // slave. We need to disconnect the switch. - // @see RoleChanger for rationale - sw.disconnectOutputStream(); - } - else { - // Controller's role is master: add to - // active - // TODO: check if clearing flow table is - // right choice here. - // Need to clear FlowMods before we add the switch - // and dispatch updates otherwise we have a race condition. - // TODO: switch update is async. Won't we still have a potential - // race condition? - addSwitch(sw, true); - } - } - } - else { - // TODO: Is this the right thing to do if we receive - // some other error besides a bad vendor error? - // Presumably that means the switch did actually - // understand the role request message, but there - // was some other error from processing the message. - // OF 1.2 specifies a OFPET_ROLE_REQUEST_FAILED - // error code, but it doesn't look like the Nicira - // role request has that. Should check OVS source - // code to see if it's possible for any other errors - // to be returned. - // If we received an error the switch is not - // in the correct role, so we need to disconnect it. - // We could also resend the request but then we need to - // check if there are other pending request in which - // case we shouldn't resend. If we do resend we need - // to make sure that the switch eventually accepts one - // of our requests or disconnect the switch. This feels - // cumbersome. - sw.disconnectOutputStream(); - } + roleChanger.deliverRoleRequestError(sw, error); } - // Once we support OF 1.2, we'd add code to handle it here. - //if (error.getXid() == state.ofRoleRequestXid) { - //} - if (shouldLogError) + else { logError(sw, error); + } break; case STATS_REPLY: if (state.hsState.ordinal() < diff --git a/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java b/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java index 16b302f1c..f9ec793b3 100644 --- a/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java +++ b/src/main/java/net/floodlightcontroller/core/internal/RoleChanger.java @@ -11,10 +11,10 @@ import java.util.concurrent.DelayQueue; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; +import org.openflow.protocol.OFError; import org.openflow.protocol.OFMessage; import org.openflow.protocol.OFType; import org.openflow.protocol.OFVendor; -import org.openflow.util.HexString; import org.openflow.vendor.nicira.OFNiciraVendorData; import org.openflow.vendor.nicira.OFRoleReplyVendorData; import org.openflow.vendor.nicira.OFRoleRequestVendorData; @@ -80,22 +80,19 @@ import net.floodlightcontroller.core.annotations.LogMessageDocs; * OFSwitchImpl's queue of pending requests) * - We handle requests and timeouts in the same thread. We use a priority queue * to schedule them so we are guaranteed that they are processed in - * the same order as they are submitted. If a request times out we drop - * the connection to this switch. + * the same order as they are submitted. If a request times out we assume + * the switch doesn't support HA role (the same as receiving an error reply). * - Since we decouple submission of role change requests and actually sending * them we cannot check a received role reply against the controller's current * role because the controller's current role could have changed again. - * - Receiving Role Reply messages is handled by OFChannelHandler and - * OFSwitchImpl directly. The OFSwitchImpl checks if the received request - * is as expected (xid and role match the head of the pending queue in - * OFSwitchImpl). If so - * the switch updates its role. Otherwise the connection is dropped. If this - * is the first reply, the SWITCH_SUPPORTS_NX_ROLE attribute is set. - * Next, we call addSwitch(), removeSwitch() to update the list of active - * switches if appropriate. + * - Receiving Role Reply messages is received by OFChannelHandler and + * delivered here. We call switch's setHARole() to mark the switch role and + * indicate that a reply was received. Next, we call addSwitch(), + * removeSwitch() to update the list of active switches if appropriate. * - If we receive an Error indicating that roles are not supported by the - * switch, we set the SWITCH_SUPPORTS_NX_ROLE to false. We keep the - * switch connection alive while in MASTER and EQUAL role. + * switch, we set the SWITCH_SUPPORTS_NX_ROLE to false. We call switch's + * setHARole(), indicating no reply was received. We keep the switch + * connection alive while in MASTER and EQUAL role. * (TODO: is this the right behavior for EQUAL??). If the role changes to * SLAVE the switch connection is dropped (remember: only if the switch * doesn't support role requests) @@ -107,13 +104,10 @@ import net.floodlightcontroller.core.annotations.LogMessageDocs; * New switch connection: * - Switch handshake is done without sending any role request messages. * - After handshake completes, switch is added to the list of connected switches - * and we send the first role request message if role - * requests are enabled. If roles are disabled automatically promote switch to - * active switch list and clear FlowTable. + * and we send the first role request message. If role is disabled, we assume + * the role is MASTER. * - When we receive the first reply we proceed as above. In addition, if - * the role request is for MASTER we wipe the flow table. We do not wipe - * the flow table if the switch connected while role supported was disabled - * on the controller. + * the role request is for MASTER we wipe the flow table. * */ public class RoleChanger { @@ -138,10 +132,6 @@ public class RoleChanger { protected static long DEFAULT_TIMEOUT = 5L*1000*1000*1000L; // 5s protected static Logger log = LoggerFactory.getLogger(RoleChanger.class); - private static final String HA_CHECK_SWITCH = - "Check the health of the indicated switch. If the problem " + - "persists or occurs repeatedly, it likely indicates a defect " + - "in the switch HA implementation."; /** * A queued task to be handled by the Role changer thread. @@ -352,12 +342,7 @@ public class RoleChanger { log.warn("Timeout while waiting for role reply from switch {}" + " with datapath {}", sw, sw.getAttribute(IOFSwitch.SWITCH_DESCRIPTION_DATA)); - sw.setHARole(entry.role, false); - if (entry.role == Role.SLAVE) { - sw.disconnectOutputStream(); - } else { - controller.addSwitch(sw, true); - } + setSwitchHARole(sw, entry.role, false); } } } @@ -369,7 +354,7 @@ public class RoleChanger { * our expectations we disconnect the switch. * * We must not check the received role against the controller's current - * role because there's no synchronization but that's fine @see RoleChanger + * role because there's no synchronization but that's fine. * * Will be called by the OFChannelHandler's receive loop * @@ -382,24 +367,24 @@ public class RoleChanger { "Role {role}" + " Disconnecting switch", explanation="The switch sent an unexpected HA role reply", - recommendation=HA_CHECK_SWITCH), + recommendation=LogMessageDoc.HA_CHECK_SWITCH), @LogMessageDoc(level="ERROR", message="Switch {switch}: expected role reply with " + "Xid {xid}, got {xid}. Disconnecting switch", explanation="The switch sent an unexpected HA role reply", - recommendation=HA_CHECK_SWITCH), + recommendation=LogMessageDoc.HA_CHECK_SWITCH), @LogMessageDoc(level="ERROR", message="Switch {switch}: expected role reply with " + "Role {role}, got {role}. Disconnecting switch", explanation="The switch sent an unexpected HA role reply", - recommendation=HA_CHECK_SWITCH) + recommendation=LogMessageDoc.HA_CHECK_SWITCH) }) protected void deliverRoleReply(IOFSwitch sw, int xid, Role role) { LinkedList<PendingRoleRequestEntry> pendingRoleRequests = pendingRequestMap.get(sw); if (pendingRoleRequests == null) { - log.warn("Switch {}: received unexpected role reply for Role {}" + + log.debug("Switch {}: received unexpected role reply for Role {}" + ", ignored", sw, role ); return; } @@ -428,7 +413,7 @@ public class RoleChanger { else { log.debug("Received role reply message from {}, setting role to {}", this, role); - sw.setHARole(role, true); + setSwitchHARole(sw, role, true); } } } @@ -486,7 +471,7 @@ public class RoleChanger { * Otherwise we ignore it. * @param xid */ - public void deliverRoleRequestNotSupported(IOFSwitch sw, int xid) { + private void deliverRoleRequestNotSupported(IOFSwitch sw, int xid) { LinkedList<PendingRoleRequestEntry> pendingRoleRequests = pendingRequestMap.get(sw); if (pendingRoleRequests == null) { @@ -497,9 +482,7 @@ public class RoleChanger { synchronized(pendingRoleRequests) { PendingRoleRequestEntry head = pendingRoleRequests.poll(); if (head != null && head.xid == xid) { - sw.setHARole(head.role, false); - } else { - sw.disconnectOutputStream(); + setSwitchHARole(sw, head.role, false); } } } @@ -622,56 +605,119 @@ public class RoleChanger { new Object[] { role, sw, controller.role} ); - Role oldRole = sw.getHARole(); deliverRoleReply(sw, vendorMessage.getXid(), role); + } + + @LogMessageDocs({ + @LogMessageDoc(level="ERROR", + message="Received ERROR from sw {switch} that indicates roles " + + "are not supported but we have received a valid role " + + "reply earlier", + explanation="Switch is responding to HA role request in an " + + "inconsistent manner.", + recommendation=LogMessageDoc.CHECK_SWITCH), + @LogMessageDoc(level="ERROR", + message="Unexpected error {error} from switch {switch} " + + "disconnecting", + explanation="Swith sent an error reply to request, but the error " + + "type is not OPET_BAD_REQUEST as required by the protocol", + recommendation=LogMessageDoc.CHECK_SWITCH) + }) + protected void deliverRoleRequestError(IOFSwitch sw, OFError error) { + // We expect to receive a bad request error when + // we're connected to a switch that doesn't support + // the Nicira vendor extensions (i.e. not OVS or + // derived from OVS). By protocol, it should also be + // BAD_VENDOR, but too many switch implementations + // get it wrong and we can already check the xid() + // so we can ignore the type with confidence that this + // is not a spurious error + boolean isBadRequestError = + (error.getErrorType() == OFError.OFErrorType. + OFPET_BAD_REQUEST.getValue()); + if (isBadRequestError) { + if (sw.getHARole() != null) { + log.warn("Received ERROR from sw {} that " + +"indicates roles are not supported " + +"but we have received a valid " + +"role reply earlier", sw); + } + // First, clean up pending requests + deliverRoleRequestNotSupported(sw, error.getXid()); + } else { + // TODO: Is this the right thing to do if we receive + // some other error besides a bad request error? + // Presumably that means the switch did actually + // understand the role request message, but there + // was some other error from processing the message. + // OF 1.2 specifies a OFPET_ROLE_REQUEST_FAILED + // error code, but it doesn't look like the Nicira + // role request has that. Should check OVS source + // code to see if it's possible for any other errors + // to be returned. + // If we received an error the switch is not + // in the correct role, so we need to disconnect it. + // We could also resend the request but then we need to + // check if there are other pending request in which + // case we shouldn't resend. If we do resend we need + // to make sure that the switch eventually accepts one + // of our requests or disconnect the switch. This feels + // cumbersome. + log.error("Unexpected error {} from switch {}, disconnecting", + error, sw); + sw.disconnectOutputStream(); + } + } + + /** + * Set switch's HA role and adjust switch's list membership + * based on the new role and switch's HA capability. This is + * a synchronized method to keep role handling consistent. + * @param sw + * @param role + * @param replyReceived + */ + private synchronized void setSwitchHARole(IOFSwitch sw, Role role, + boolean replyReceived) { + // Record previous role and set the current role + // We tell switch whether a correct reply was received. + // The switch may deduce if HA role request is supported + // (if it doesn't already know). + Role oldRole = sw.getHARole(); + sw.setHARole(role, replyReceived); - if (sw.getHARole() != Role.SLAVE) { - // Transition from SLAVE to MASTER. - boolean shouldClearFlowMods = + // Adjust the active switch list based on the role + if (role != Role.SLAVE) { + // SLAVE->MASTER, need to add switch to active list. + // If this is the initial connection, clear flodmods. + boolean clearFlowMods = (oldRole == null) || controller.getAlwaysClearFlowsOnSwAdd(); - if (oldRole == null) { - // This is the first role-reply message we receive from - // this switch. Delete all pre-existing flows for new - // connections to the master. - // - // FIXME: Need to think more about what the test should - // be for when we flush the flow-table? For example, - // if all the controllers are temporarily in the backup - // role (e.g. right after a failure of the master - // controller) at the point the switch connects, then - // all of the controllers will initially connect as - // backup controllers and not flush the flow-table. - // Then when one of them is promoted to master following - // the master controller election the flow-table - // will still not be flushed because that's treated as - // a failover event where we don't want to flush the - // flow-table. The end result would be that the flow - // table for a newly connected switch is never - // flushed. Not sure how to handle that case though... - shouldClearFlowMods = true; - log.debug("First role reply from master switch {}, " + - "clear FlowTable to active switch list", - HexString.toHexString(sw.getId())); + controller.addSwitch(sw, clearFlowMods); + } else { + // Initial SLAVE setting. Nothing to do if role reply received. + // Else disconnect the switch. + if (oldRole == null && !replyReceived) { + sw.disconnectOutputStream(); + } else { + // MASTER->SLAVE transition. If switch supports HA roles, + // remove it from active list, but still leave it connected. + // Otherwise, disconnect it. Cleanup happens after channel + // handler receives the disconnect notification. + if (replyReceived) { + controller.removeSwitch(sw); + } else { + sw.disconnectOutputStream(); + } } - - // Only add the switch to the active switch list if - // we're not in the slave role. Note that if the role - // attribute is null, then that means that the switch - // doesn't support the role request messages, so in that - // case we're effectively in the EQUAL role and the - // switch should be included in the active switch list. - controller.addSwitch(sw, shouldClearFlowMods); - log.debug("Added master switch {} to active switch list", - HexString.toHexString(sw.getId())); - } - else { - // Transition from MASTER to SLAVE: remove switch - // from active switch list. - log.debug("Removed slave switch {} from active switch" + - " list", HexString.toHexString(sw.getId())); - controller.removeSwitch(sw); } } - + /** + * Cleanup pending requests associated witch switch. Called when + * switch disconnects. + * @param sw + */ + public void removePendingRequests(IOFSwitch sw) { + pendingRequestMap.remove(sw); + } } diff --git a/src/test/java/net/floodlightcontroller/core/internal/ControllerTest.java b/src/test/java/net/floodlightcontroller/core/internal/ControllerTest.java index 5071fff1e..bb9d90ce0 100644 --- a/src/test/java/net/floodlightcontroller/core/internal/ControllerTest.java +++ b/src/test/java/net/floodlightcontroller/core/internal/ControllerTest.java @@ -70,7 +70,6 @@ import org.easymock.EasyMock; import org.jboss.netty.channel.Channel; import org.junit.Test; import org.openflow.protocol.OFError; -import org.openflow.protocol.OFError.OFBadRequestCode; import org.openflow.protocol.OFError.OFErrorType; import org.openflow.protocol.OFFeaturesReply; import org.openflow.protocol.OFPacketIn; @@ -1029,7 +1028,6 @@ public class ControllerTest extends FloodlightTestCase msg.setType(OFType.ERROR); msg.setXid(xid); msg.setErrorType(OFErrorType.OFPET_BAD_REQUEST); - msg.setErrorCode(OFBadRequestCode.OFPBRC_BAD_VENDOR); // the switch connection should get disconnected when the controller is // in SLAVE mode and the switch does not support role-request messages @@ -1037,6 +1035,7 @@ public class ControllerTest extends FloodlightTestCase setupPendingRoleRequest(chdlr.sw, xid, controller.role, 123456); expect(chdlr.sw.getHARole()).andReturn(null); chdlr.sw.setHARole(Role.SLAVE, false); + expect(chdlr.sw.getHARole()).andReturn(Role.SLAVE); chdlr.sw.disconnectOutputStream(); replay(ch, chdlr.sw); @@ -1046,24 +1045,6 @@ public class ControllerTest extends FloodlightTestCase controller.activeSwitches.isEmpty()); reset(ch, chdlr.sw); - - // a different error message - should also reject role request - msg.setErrorType(OFErrorType.OFPET_BAD_REQUEST); - msg.setErrorCode(OFBadRequestCode.OFPBRC_EPERM); - controller.role = Role.SLAVE; - setupPendingRoleRequest(chdlr.sw, xid, controller.role, 123456); - expect(chdlr.sw.getHARole()).andReturn(null); - chdlr.sw.setHARole(Role.SLAVE, false); - chdlr.sw.disconnectOutputStream(); - replay(ch, chdlr.sw); - - chdlr.processOFMessage(msg); - verify(ch, chdlr.sw); - assertTrue("activeSwitches must be empty", - controller.activeSwitches.isEmpty()); - reset(ch, chdlr.sw); - - // We are MASTER, the switch should be added to the list of active // switches. controller.role = Role.MASTER; diff --git a/src/test/java/net/floodlightcontroller/core/internal/RoleChangerTest.java b/src/test/java/net/floodlightcontroller/core/internal/RoleChangerTest.java index 507768959..cb446b2a9 100644 --- a/src/test/java/net/floodlightcontroller/core/internal/RoleChangerTest.java +++ b/src/test/java/net/floodlightcontroller/core/internal/RoleChangerTest.java @@ -27,6 +27,8 @@ import org.easymock.EasyMock; import org.jboss.netty.channel.Channel; import org.junit.Before; import org.junit.Test; +import org.openflow.protocol.OFError; +import org.openflow.protocol.OFError.OFErrorType; import org.openflow.protocol.OFMessage; import org.openflow.protocol.OFType; import org.openflow.protocol.OFVendor; @@ -182,6 +184,7 @@ public class RoleChangerTest { // Timed out switch should become active expect(sw2.getAttribute(IOFSwitch.SWITCH_DESCRIPTION_DATA)) .andReturn(null); + expect(sw2.getHARole()).andReturn(null); sw2.setHARole(Role.MASTER, false); EasyMock.expectLastCall(); switches.add(sw2); @@ -239,9 +242,11 @@ public class RoleChangerTest { (FloodlightContext)EasyMock.anyObject()); expect(sw1.getAttribute(IOFSwitch.SWITCH_DESCRIPTION_DATA)) .andReturn(null); + expect(sw1.getHARole()).andReturn(null); sw1.setHARole(Role.MASTER, false); expect(sw1.getAttribute(IOFSwitch.SWITCH_DESCRIPTION_DATA)) .andReturn(null); + expect(sw1.getHARole()).andReturn(Role.MASTER); sw1.setHARole(Role.SLAVE, false); // Disconnect on timing out SLAVE request sw1.disconnectOutputStream(); @@ -402,33 +407,47 @@ public class RoleChangerTest { } @Test - public void testDeliverRoleRequestNotSupported () { + public void testDeliverRoleRequestError() { // normal case. xid is pending int xid = (int) System.currentTimeMillis(); long cookie = System.nanoTime(); Role role = Role.MASTER; OFSwitchImpl sw = new OFSwitchImpl(); + Channel ch = createMock(Channel.class); + SocketAddress sa = new InetSocketAddress(42); + expect(ch.getRemoteAddress()).andReturn(sa).anyTimes(); + sw.setChannel(ch); setupPendingRoleRequest(sw, xid, role, cookie); - roleChanger.deliverRoleRequestNotSupported(sw, xid); + OFError error = new OFError(); + error.setErrorType(OFErrorType.OFPET_BAD_REQUEST); + error.setXid(xid); + replay(ch); + roleChanger.deliverRoleRequestError(sw, error); + verify(ch); assertEquals(false, sw.getAttribute(IOFSwitch.SWITCH_SUPPORTS_NX_ROLE)); assertEquals(role, sw.getHARole()); assertEquals(0, roleChanger.pendingRequestMap.get(sw).size()); } @Test - public void testDeliverRoleRequestNotSupportedNonePending() { + public void testDeliverRoleRequestErrorNonePending() { // nothing pending OFSwitchImpl sw = new OFSwitchImpl(); Channel ch = createMock(Channel.class); SocketAddress sa = new InetSocketAddress(42); expect(ch.getRemoteAddress()).andReturn(sa).anyTimes(); sw.setChannel(ch); - roleChanger.deliverRoleRequestNotSupported(sw, 1); + OFError error = new OFError(); + error.setErrorType(OFErrorType.OFPET_BAD_REQUEST); + error.setXid(1); + replay(ch); + roleChanger.deliverRoleRequestError(sw, error); + verify(ch); assertEquals(null, sw.getHARole()); } @Test - public void testDeliverRoleRequestNotSupportedWrongXid() { + public void testDeliverRoleRequestErrorWrongXid() { // wrong xid received // wrong xid received int xid = (int) System.currentTimeMillis(); @@ -439,13 +458,16 @@ public class RoleChangerTest { Channel ch = createMock(Channel.class); SocketAddress sa = new InetSocketAddress(42); expect(ch.getRemoteAddress()).andReturn(sa).anyTimes(); - sw.setChannel(ch); expect(ch.close()).andReturn(null); + sw.setChannel(ch); replay(ch); - roleChanger.deliverRoleRequestNotSupported(sw, xid+1); + OFError error = new OFError(); + error.setErrorCode(OFError.OFErrorType.OFPET_BAD_REQUEST.getValue()); + error.setXid(xid + 1); + roleChanger.deliverRoleRequestError(sw, error); verify(ch); assertEquals(null, sw.getAttribute(IOFSwitch.SWITCH_SUPPORTS_NX_ROLE)); - assertEquals(0, roleChanger.pendingRequestMap.get(sw).size()); + assertEquals(1, roleChanger.pendingRequestMap.get(sw).size()); } public void doSendNxRoleRequest(Role role, int nx_role) throws Exception { -- GitLab