Skip to content
Snippets Groups Projects
Commit 9bd9334f authored by Reynold Xin's avatar Reynold Xin Committed by Aaron Davidson
Browse files

Config updates for the new shuffle transport.

Author: Reynold Xin <rxin@databricks.com>

Closes #3657 from rxin/conf-update and squashes the following commits:

7370eab [Reynold Xin] Config updates for the new shuffle transport.
parent 2b9b7268
No related branches found
No related tags found
No related merge requests found
......@@ -35,14 +35,14 @@ public class TransportConf {
return conf.getBoolean("spark.shuffle.io.preferDirectBufs", true);
}
/** Connect timeout in secs. Default 120 secs. */
/** Connect timeout in milliseconds. Default 120 secs. */
public int connectionTimeoutMs() {
return conf.getInt("spark.shuffle.io.connectionTimeout", 120) * 1000;
}
/** Number of concurrent connections between two nodes for fetching data. **/
public int numConnectionsPerPeer() {
return conf.getInt("spark.shuffle.io.numConnectionsPerPeer", 2);
return conf.getInt("spark.shuffle.io.numConnectionsPerPeer", 1);
}
/** Requested maximum length of the queue of incoming connections. Default -1 for no backlog. */
......@@ -67,7 +67,7 @@ public class TransportConf {
public int sendBuf() { return conf.getInt("spark.shuffle.io.sendBuffer", -1); }
/** Timeout for a single round trip of SASL token exchange, in milliseconds. */
public int saslRTTimeout() { return conf.getInt("spark.shuffle.sasl.timeout", 30000); }
public int saslRTTimeoutMs() { return conf.getInt("spark.shuffle.sasl.timeout", 30) * 1000; }
/**
* Max number of times we will try IO exceptions (such as connection timeouts) per request.
......@@ -79,7 +79,7 @@ public class TransportConf {
* Time (in milliseconds) that we will wait in order to perform a retry after an IOException.
* Only relevant if maxIORetries &gt; 0.
*/
public int ioRetryWaitTime() { return conf.getInt("spark.shuffle.io.retryWaitMs", 5000); }
public int ioRetryWaitTimeMs() { return conf.getInt("spark.shuffle.io.retryWait", 5) * 1000; }
/**
* Minimum size of a block that we should start using memory map rather than reading in through
......
......@@ -59,7 +59,7 @@ public class SaslClientBootstrap implements TransportClientBootstrap {
ByteBuf buf = Unpooled.buffer(msg.encodedLength());
msg.encode(buf);
byte[] response = client.sendRpcSync(buf.array(), conf.saslRTTimeout());
byte[] response = client.sendRpcSync(buf.array(), conf.saslRTTimeoutMs());
payload = saslClient.response(response);
}
} finally {
......
......@@ -106,7 +106,7 @@ public class RetryingBlockFetcher {
this.fetchStarter = fetchStarter;
this.listener = listener;
this.maxRetries = conf.maxIORetries();
this.retryWaitTime = conf.ioRetryWaitTime();
this.retryWaitTime = conf.ioRetryWaitTimeMs();
this.outstandingBlocksIds = Sets.newLinkedHashSet();
Collections.addAll(outstandingBlocksIds, blockIds);
this.currentListener = new RetryingBlockFetchListener();
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment