diff --git a/src/main/java/net/floodlightcontroller/core/IOFSwitch.java b/src/main/java/net/floodlightcontroller/core/IOFSwitch.java index 6bc92c98fae1e9866c99606d352f6115d1b0c618..388349f2bf46a24a402ddb277abbebe140e2f77e 100644 --- a/src/main/java/net/floodlightcontroller/core/IOFSwitch.java +++ b/src/main/java/net/floodlightcontroller/core/IOFSwitch.java @@ -106,6 +106,22 @@ public interface IOFSwitch { */ public void setChannel(Channel channel); + /** + * Called when OFMessage enters pipeline. Returning true cause the message + * to be dropped. + * @param ofm + * @return + */ + public boolean inputThrottled(OFMessage ofm); + + /** + * Return if the switch is currently overloaded. The definition of + * overload refers to excessive traffic in the control path, namely + * a high packet in rate. + * @return + */ + boolean isOverloaded(); + /** * Write OFMessage to the output stream, subject to switch rate limiting. * The message will be handed to the floodlightProvider for possible filtering diff --git a/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java b/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java index 5e26c58b27979bbada886b7995f43b8cb331f4f9..351e3717b7bdb4ab12a26d3964a1cf0f4f4f369e 100644 --- a/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java +++ b/src/main/java/net/floodlightcontroller/core/OFSwitchBase.java @@ -20,6 +20,7 @@ package net.floodlightcontroller.core; import java.io.IOException; import java.net.SocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -41,7 +42,11 @@ import net.floodlightcontroller.core.internal.Controller; import net.floodlightcontroller.core.internal.OFFeaturesReplyFuture; import net.floodlightcontroller.core.internal.OFStatisticsFuture; import net.floodlightcontroller.core.web.serializers.DPIDSerializer; +import net.floodlightcontroller.devicemanager.SwitchPort; +import net.floodlightcontroller.packet.Ethernet; +import net.floodlightcontroller.routing.ForwardingBase; import net.floodlightcontroller.threadpool.IThreadPoolService; +import net.floodlightcontroller.util.MACAddress; import net.floodlightcontroller.util.TimedCache; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -54,6 +59,7 @@ import org.openflow.protocol.OFFeaturesReply; import org.openflow.protocol.OFFlowMod; import org.openflow.protocol.OFMatch; import org.openflow.protocol.OFMessage; +import org.openflow.protocol.OFPacketIn; import org.openflow.protocol.OFPhysicalPort; import org.openflow.protocol.OFPhysicalPort.OFPortConfig; import org.openflow.protocol.OFPhysicalPort.OFPortState; @@ -113,6 +119,20 @@ public abstract class OFSwitchBase implements IOFSwitch { // Private members for throttling private boolean writeThrottleEnabled = false; + protected boolean packetInThrottleEnabled = false; // used by test + private int packetInRateThresholdHigh = Integer.MAX_VALUE; + private int packetInRateThresholdLow = 1; + private int packetInRatePerMacThreshold = Integer.MAX_VALUE; + private int packetInRatePerPortThreshold = Integer.MAX_VALUE; + private long messageCount = 0; + private long messageCountUniqueOFMatch = 0; + private long lastMessageTime; + private int currentRate = 0; + private TimedCache<OFMatch> ofMatchCache; + private TimedCache<Long> macCache; + private TimedCache<Long> macBlockedCache; + private TimedCache<Short> portCache; + private TimedCache<Short> portBlockedCache; protected final static ThreadLocal<Map<IOFSwitch,List<OFMessage>>> local_msg_buffer = new ThreadLocal<Map<IOFSwitch,List<OFMessage>>>() { @@ -141,6 +161,7 @@ public abstract class OFSwitchBase implements IOFSwitch { this.timedCache = new TimedCache<Long>(100, 5*1000 ); // 5 seconds interval this.listenerLock = new ReentrantReadWriteLock(); this.portBroadcastCacheHitMap = new ConcurrentHashMap<Short, AtomicLong>(); + this.lastMessageTime = System.currentTimeMillis(); // Defaults properties for an ideal switch this.setAttribute(PROP_FASTWILDCARDS, OFMatch.OFPFW_ALL); @@ -748,4 +769,186 @@ public abstract class OFSwitchBase implements IOFSwitch { public void setFloodlightProvider(Controller controller) { floodlightProvider = controller; } + + + /** + * For switch drivers to set thresholds, all rates in per second + * @param pktInHigh - above this start throttling + * @param pktInLow - below this stop throttling + * @param pktInPerMac - block host if unique pktIn rate reaches this + * @param pktInPerPort - block port if unique pktIn rate reaches this + */ + @JsonIgnore + protected void setInputThrottleThresholds(int pktInHigh, int pktInLow, + int pktInPerMac, int pktInPerPort) { + packetInRateThresholdHigh = pktInHigh; + packetInRateThresholdLow = pktInLow; + packetInRatePerMacThreshold = pktInPerMac; + packetInRatePerPortThreshold = pktInPerPort; + } + + /** + * Return if switch has exceeded the high threshold of packet in rate. + * @return + */ + @Override + public boolean isOverloaded() { + return packetInThrottleEnabled; + } + + /** + * Determine if this message should be dropped. + * + * We compute the current rate by taking a timestamp every 100 messages. + * Could change to a more complex scheme if more accuracy is needed. + * + * Enable throttling if the rate goes above packetInRateThresholdHigh + * Disable throttling when the rate drops below packetInRateThresholdLow + * + * While throttling is enabled, we do the following: + * - Remove duplicate packetIn's mapped to the same OFMatch + * - After filtering, if packetIn rate per host (mac) is above + * packetInRatePerMacThreshold, push a flow mod to block mac on port + * - After filtering, if packetIn rate per port is above + * packetInRatePerPortThreshold, push a flow mod to block port + * - Allow blocking flow mods have a hard timeout and expires automatically + * + * TODO: keep a history of all events related in input throttling + * + * @param ofm + * @return + */ + @Override + public boolean inputThrottled(OFMessage ofm) { + if (ofm.getType() != OFType.PACKET_IN) { + return false; + } + // Compute current packet in rate + messageCount++; + if (messageCount % 100 == 0) { + long now = System.currentTimeMillis(); + if (now != lastMessageTime) { + currentRate = (int) (100000 / (now - lastMessageTime)); + lastMessageTime = now; + } else { + currentRate = Integer.MAX_VALUE; + } + } + if (!packetInThrottleEnabled) { + if (currentRate <= packetInRateThresholdHigh) { + return false; // most common case + } + enablePacketInThrottle(); + } else if (currentRate < packetInRateThresholdLow) { + disablePacketInThrottle(); + return false; + } + + // Now we are in the slow path where we need to do filtering + // First filter based on OFMatch + OFPacketIn pin = (OFPacketIn)ofm; + OFMatch match = new OFMatch(); + match.loadFromPacket(pin.getPacketData(), pin.getInPort()); + if (ofMatchCache.update(match)) { + // TODO keep stats for dropped packets + return true; + } + + // We have packet in with a distinct flow, check per mac rate + messageCountUniqueOFMatch++; + if ((messageCountUniqueOFMatch % packetInRatePerMacThreshold) == 1) { + checkPerSourceMacRate(pin); + } + + // Check per port rate + if ((messageCountUniqueOFMatch % packetInRatePerPortThreshold) == 1) { + checkPerPortRate(pin); + } + return false; + } + + /** + * We rely on the fact that packet in processing is single threaded + * per switch, so no locking is necessary. + */ + private void disablePacketInThrottle() { + ofMatchCache = null; + macCache = null; + macBlockedCache = null; + portCache = null; + portBlockedCache = null; + packetInThrottleEnabled = false; + log.info("Packet in rate is {}, disable throttling on {}", + currentRate, this); + } + + private void enablePacketInThrottle() { + ofMatchCache = new TimedCache<OFMatch>(2048, 5000); // 5 second interval + macCache = new TimedCache<Long>(64, 1000 ); // remember last second + macBlockedCache = new TimedCache<Long>(256, 5000 ); // 5 second interval + portCache = new TimedCache<Short>(16, 1000 ); // rememeber last second + portBlockedCache = new TimedCache<Short>(64, 5000 ); // 5 second interval + packetInThrottleEnabled = true; + messageCountUniqueOFMatch = 0; + log.info("Packet in rate is {}, enable throttling on {}", + currentRate, this); + } + + /** + * Check if we have sampled this mac in the last second. + * Since we check every packetInRatePerMacThreshold packets, + * the presence of the mac in the macCache means the rate is + * above the threshold in a statistical sense. + * + * Take care not to block topology probing packets. Also don't + * push blocking flow mod if we have already done so within the + * last 5 seconds. + * + * @param pin + * @return + */ + private void checkPerSourceMacRate(OFPacketIn pin) { + byte[] data = pin.getPacketData(); + byte[] mac = Arrays.copyOfRange(data, 6, 12); + MACAddress srcMac = MACAddress.valueOf(mac); + short ethType = (short) (((data[12] & 0xff) << 8) + (data[13] & 0xff)); + if (ethType != Ethernet.TYPE_LLDP && ethType != Ethernet.TYPE_BSN && + macCache.update(srcMac.toLong())) { + // Check if we already pushed a flow in the last 5 seconds + if (macBlockedCache.update(srcMac.toLong())) { + return; + } + // write out drop flow per srcMac + int port = pin.getInPort(); + SwitchPort swPort = new SwitchPort(getId(), port); + ForwardingBase.blockHost(floodlightProvider, + swPort, srcMac.toLong(), (short) 5, 0); + log.info("Excessive packet in from {} on {}, block host for 5 sec", + srcMac.toString(), swPort); + } + } + + /** + * Works in a similar way as checkPerSourceMacRate(). + * + * TODO Don't block ports with links? + * + * @param pin + * @return + */ + private void checkPerPortRate(OFPacketIn pin) { + Short port = pin.getInPort(); + if (portCache.update(port)) { + // Check if we already pushed a flow in the last 5 seconds + if (portBlockedCache.update(port)) { + return; + } + // write out drop flow per port + SwitchPort swPort = new SwitchPort(getId(), port); + ForwardingBase.blockHost(floodlightProvider, + swPort, -1L, (short) 5, 0); + log.info("Excessive packet in from {}, block port for 5 sec", + swPort); + } + } } diff --git a/src/main/java/net/floodlightcontroller/core/internal/Controller.java b/src/main/java/net/floodlightcontroller/core/internal/Controller.java index 77ee0e1d10c5b7dad37df8e80c064be96b493731..8999f92ed4af1bfc964384d67d07abf5132b74c3 100644 --- a/src/main/java/net/floodlightcontroller/core/internal/Controller.java +++ b/src/main/java/net/floodlightcontroller/core/internal/Controller.java @@ -613,6 +613,10 @@ public class Controller implements IFloodlightProviderService, } for (OFMessage ofm : msglist) { + // Per-switch input throttling + if (sw != null && sw.inputThrottled(ofm)) { + continue; + } try { if (overload_drop && !loadlevel.equals(LoadMonitor.LoadLevel.OK)) { diff --git a/src/main/java/net/floodlightcontroller/routing/ForwardingBase.java b/src/main/java/net/floodlightcontroller/routing/ForwardingBase.java index 9be2079cf1aa2cf268cc14c6b5fa790439a7b3d8..11478f46bbdce6d2709d252a3a2c9cdef613cb84 100644 --- a/src/main/java/net/floodlightcontroller/routing/ForwardingBase.java +++ b/src/main/java/net/floodlightcontroller/routing/ForwardingBase.java @@ -554,7 +554,7 @@ public abstract class ForwardingBase if (sw == null) return false; int inputPort = sw_tup.getPort(); log.debug("blockHost sw={} port={} mac={}", - new Object[] { sw, sw_tup.getPort(), new Long(host_mac) }); + new Object[] { sw, sw_tup.getPort(), Long.valueOf(host_mac) }); // Create flow-mod based on packet-in and src-switch OFFlowMod fm = @@ -563,10 +563,14 @@ public abstract class ForwardingBase OFMatch match = new OFMatch(); List<OFAction> actions = new ArrayList<OFAction>(); // Set no action to // drop - match.setDataLayerSource(Ethernet.toByteArray(host_mac)) - .setInputPort((short)inputPort) - .setWildcards(OFMatch.OFPFW_ALL & ~OFMatch.OFPFW_DL_SRC - & ~OFMatch.OFPFW_IN_PORT); + match.setInputPort((short)inputPort); + if (host_mac != -1L) { + match.setDataLayerSource(Ethernet.toByteArray(host_mac)) + .setWildcards(OFMatch.OFPFW_ALL & ~OFMatch.OFPFW_DL_SRC + & ~OFMatch.OFPFW_IN_PORT); + } else { + match.setWildcards(OFMatch.OFPFW_ALL & ~OFMatch.OFPFW_IN_PORT); + } fm.setCookie(cookie) .setHardTimeout(hardTimeout) .setIdleTimeout(FLOWMOD_DEFAULT_IDLE_TIMEOUT) diff --git a/src/test/java/net/floodlightcontroller/core/internal/OFSwitchBaseTest.java b/src/test/java/net/floodlightcontroller/core/internal/OFSwitchBaseTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4ef843c7a170d8b2f5ee9238551bcc5e9a713277 --- /dev/null +++ b/src/test/java/net/floodlightcontroller/core/internal/OFSwitchBaseTest.java @@ -0,0 +1,285 @@ +/** + * Copyright 2013, Big Switch Networks, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + **/ + +package net.floodlightcontroller.core.internal; + +import static org.easymock.EasyMock.*; +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import net.floodlightcontroller.core.FloodlightContext; +import net.floodlightcontroller.core.IFloodlightProviderService; +import net.floodlightcontroller.core.IOFSwitch; +import net.floodlightcontroller.core.OFSwitchBase; +import net.floodlightcontroller.packet.ARP; +import net.floodlightcontroller.packet.Ethernet; +import net.floodlightcontroller.packet.IPacket; +import net.floodlightcontroller.packet.IPv4; +import org.junit.Before; +import org.junit.Test; +import org.openflow.protocol.OFFlowMod; +import org.openflow.protocol.OFMatch; +import org.openflow.protocol.OFMessage; +import org.openflow.protocol.OFPacketIn; +import org.openflow.protocol.OFType; +import org.openflow.protocol.OFPacketIn.OFPacketInReason; +import org.openflow.protocol.factory.BasicFactory; +import org.openflow.protocol.statistics.OFDescriptionStatistics; +import org.openflow.util.HexString; + +public class OFSwitchBaseTest { + private static final String srcMac = "00:44:33:22:11:00"; + IFloodlightProviderService floodlightProvider; + Map<Long, IOFSwitch> switches; + private OFMessage blockMessage; + private OFPacketIn pi; + private IPacket testPacket; + private byte[] testPacketSerialized; + + private class OFSwitchTest extends OFSwitchBase { + public OFSwitchTest(IFloodlightProviderService fp) { + super(); + stringId = "whatever"; + datapathId = 1L; + floodlightProvider = fp; + } + + @Override + public void setSwitchProperties(OFDescriptionStatistics description) { + // TODO Auto-generated method stub + } + + @Override + public OFPortType getPortType(short port_num) { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isFastPort(short port_num) { + // TODO Auto-generated method stub + return false; + } + + @Override + public List<Short> getUplinkPorts() { + // TODO Auto-generated method stub + return null; + } + + public void write(OFMessage msg, FloodlightContext cntx) { + blockMessage = msg; + } + + public void setThresholds(int high, int low, int host, int port) { + sw.setInputThrottleThresholds(high, low, host, port); + } + + public boolean inputThrottleEnabled() { + return packetInThrottleEnabled; + } + + @Override + public String toString() { + return "OFSwitchTest"; + } + } + private OFSwitchTest sw; + + @Before + public void setUp() throws Exception { + blockMessage = null; + // Build our test packet + testPacket = new Ethernet() + .setSourceMACAddress(srcMac) + .setDestinationMACAddress("00:11:22:33:44:55") + .setEtherType(Ethernet.TYPE_ARP) + .setPayload( + new ARP() + .setHardwareType(ARP.HW_TYPE_ETHERNET) + .setProtocolType(ARP.PROTO_TYPE_IP) + .setHardwareAddressLength((byte) 6) + .setProtocolAddressLength((byte) 4) + .setOpCode(ARP.OP_REPLY) + .setSenderHardwareAddress(Ethernet.toMACAddress("00:44:33:22:11:00")) + .setSenderProtocolAddress(IPv4.toIPv4AddressBytes("192.168.1.1")) + .setTargetHardwareAddress(Ethernet.toMACAddress("00:11:22:33:44:55")) + .setTargetProtocolAddress(IPv4.toIPv4AddressBytes("192.168.1.2"))); + testPacketSerialized = testPacket.serialize(); + + pi = ((OFPacketIn) BasicFactory.getInstance().getMessage(OFType.PACKET_IN)) + .setBufferId(-1) + .setInPort((short) 1) + .setPacketData(testPacketSerialized) + .setReason(OFPacketInReason.NO_MATCH) + .setTotalLength((short) testPacketSerialized.length); + floodlightProvider = createMock(IFloodlightProviderService.class); + sw = new OFSwitchTest(floodlightProvider); + switches = new ConcurrentHashMap<Long, IOFSwitch>(); + switches.put(sw.getId(), sw); + expect(floodlightProvider.getSwitches()).andReturn(switches).anyTimes(); + expect(floodlightProvider.getOFMessageFactory()) + .andReturn(BasicFactory.getInstance()).anyTimes(); + replay(floodlightProvider); + } + + /** + * By default, high threshold is infinite + */ + @Test + public void testNoPacketInThrottle() { + for (int i = 0; i < 200; i++) { + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(blockMessage == null); + assertFalse(sw.inputThrottleEnabled()); + } + + /** + * The test sends packet in at infinite rate (< 1ms), + * so throttling should be enabled on 100th packet, when the first + * rate measurement is done. + */ + @Test + public void testPacketInStartThrottle() { + int high = 500; + sw.setThresholds(high, 10, 50, 200); + // We measure time lapse every 100 packets + for (int i = 0; i < 100; i++) { + assertFalse(sw.inputThrottleEnabled()); + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(sw.inputThrottleEnabled()); + assertTrue(sw.inputThrottled(pi)); + assertTrue(sw.inputThrottled(pi)); + assertTrue(blockMessage == null); + } + + /** + * With throttling enabled, raise the low water mark threshold, + * verify throttling stops. + * @throws InterruptedException + */ + @Test + public void testPacketInStopThrottle() throws InterruptedException { + sw.setThresholds(100, 10, 50, 200); + // First, enable throttling + for (int i = 0; i < 100; i++) { + assertFalse(sw.inputThrottleEnabled()); + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(sw.inputThrottleEnabled()); + + sw.setThresholds(Integer.MAX_VALUE, 100000, 50, 200); + for (int i = 0; i < 99; i++) { + assertTrue(sw.inputThrottled(pi)); + assertTrue(sw.inputThrottleEnabled()); + } + // Sleep for 2 msec, next packet should disable throttling + Thread.sleep(2); + assertFalse(sw.inputThrottled(pi)); + assertFalse(sw.inputThrottleEnabled()); + } + + /** + * With throttling enabled, if rate of unique flows from a host + * exceeds set threshold, a flow mod should be emitted to block host + */ + @Test + public void testPacketInBlockHost() { + int high = 500; + int perMac = 50; + sw.setThresholds(high, 10, perMac, 200); + // First, enable throttling + for (int i = 0; i < 100; i++) { + assertFalse(sw.inputThrottleEnabled()); + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(sw.inputThrottleEnabled()); + assertTrue(blockMessage == null); + + // Build unique flows with the same source mac + for (int j = 0; j < perMac - 1; j++) { + testPacketSerialized[5]++; + pi.setPacketData(testPacketSerialized); + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(blockMessage == null); + testPacketSerialized[5]++; + pi.setPacketData(testPacketSerialized); + assertFalse(sw.inputThrottled(pi)); + + // Verify the message is a flowmod with a hard timeout and srcMac + assertTrue(blockMessage != null); + assertTrue(blockMessage instanceof OFFlowMod); + OFFlowMod fm = (OFFlowMod) blockMessage; + assertTrue(fm.getHardTimeout() == 5); + OFMatch match = fm.getMatch(); + assertTrue((match.getWildcards() & OFMatch.OFPFW_DL_SRC) == 0); + assertTrue(Arrays.equals(match.getDataLayerSource(), + HexString.fromHexString(srcMac))); + + // Verify non-unique OFMatches are throttled + assertTrue(sw.inputThrottled(pi)); + } + + /** + * With throttling enabled, if rate of unique flows from a port + * exceeds set threshold, a flow mod should be emitted to block port + */ + @Test + public void testPacketInBlockPort() { + int high = 500; + int perPort = 200; + sw.setThresholds(high, 10, 50, perPort); + // First, enable throttling + for (int i = 0; i < 100; i++) { + assertFalse(sw.inputThrottleEnabled()); + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(sw.inputThrottleEnabled()); + assertTrue(blockMessage == null); + + // Build unique flows with different source mac + for (int j = 0; j < perPort - 1; j++) { + testPacketSerialized[11]++; + pi.setPacketData(testPacketSerialized); + assertFalse(sw.inputThrottled(pi)); + } + assertTrue(blockMessage == null); + testPacketSerialized[11]++; + pi.setPacketData(testPacketSerialized); + assertFalse(sw.inputThrottled(pi)); + + // Verify the message is a flowmod with a hard timeout and per port + assertTrue(blockMessage != null); + assertTrue(blockMessage instanceof OFFlowMod); + OFFlowMod fm = (OFFlowMod) blockMessage; + assertTrue(fm.getHardTimeout() == 5); + OFMatch match = fm.getMatch(); + assertTrue((match.getWildcards() & OFMatch.OFPFW_DL_SRC) != 0); + assertTrue((match.getWildcards() & OFMatch.OFPFW_IN_PORT) == 0); + assertTrue(match.getInputPort() == 1); + + // Verify non-unique OFMatches are throttled + assertTrue(sw.inputThrottled(pi)); + } + +} diff --git a/src/test/java/net/floodlightcontroller/util/OFMessageDamperMockSwitch.java b/src/test/java/net/floodlightcontroller/util/OFMessageDamperMockSwitch.java index 29191365cddad3344d7390f4d363cfcd0eaf8820..1d1ff169b99bf5ac4de2ce881ee5b4184e32bf7a 100644 --- a/src/test/java/net/floodlightcontroller/util/OFMessageDamperMockSwitch.java +++ b/src/test/java/net/floodlightcontroller/util/OFMessageDamperMockSwitch.java @@ -433,4 +433,16 @@ public class OFMessageDamperMockSwitch implements IOFSwitch { return false; } + @Override + public boolean inputThrottled(OFMessage ofm) { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean isOverloaded() { + // TODO Auto-generated method stub + return false; + } + }