diff --git a/core/src/main/scala/org/apache/spark/Accumulator.scala b/core/src/main/scala/org/apache/spark/Accumulator.scala
index 9d1f1d59dbce1ca93a0ec089858b1aeafa76bca9..7bea636c94aa03b106533b5eabbef77220c2be9d 100644
--- a/core/src/main/scala/org/apache/spark/Accumulator.scala
+++ b/core/src/main/scala/org/apache/spark/Accumulator.scala
@@ -26,7 +26,7 @@ package org.apache.spark
*
* An accumulator is created from an initial value `v` by calling
* [[SparkContext#accumulator SparkContext.accumulator]].
- * Tasks running on the cluster can then add to it using the [[Accumulable#+= +=]] operator.
+ * Tasks running on the cluster can then add to it using the `+=` operator.
* However, they cannot read its value. Only the driver program can read the accumulator's value,
* using its [[#value]] method.
*
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 04d657c09afd0ef565f14016014ce9d4f2d4454e..0c1c68de89f81981bf2fb4a931d7857d97a8782b 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -262,7 +262,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
/**
* Get a time parameter as seconds; throws a NoSuchElementException if it's not set. If no
* suffix is provided then seconds are assumed.
- * @throws NoSuchElementException
+ * @throws java.util.NoSuchElementException
*/
def getTimeAsSeconds(key: String): Long = {
Utils.timeStringAsSeconds(get(key))
@@ -279,7 +279,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
/**
* Get a time parameter as milliseconds; throws a NoSuchElementException if it's not set. If no
* suffix is provided then milliseconds are assumed.
- * @throws NoSuchElementException
+ * @throws java.util.NoSuchElementException
*/
def getTimeAsMs(key: String): Long = {
Utils.timeStringAsMs(get(key))
@@ -296,7 +296,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
/**
* Get a size parameter as bytes; throws a NoSuchElementException if it's not set. If no
* suffix is provided then bytes are assumed.
- * @throws NoSuchElementException
+ * @throws java.util.NoSuchElementException
*/
def getSizeAsBytes(key: String): Long = {
Utils.byteStringAsBytes(get(key))
@@ -320,7 +320,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
/**
* Get a size parameter as Kibibytes; throws a NoSuchElementException if it's not set. If no
* suffix is provided then Kibibytes are assumed.
- * @throws NoSuchElementException
+ * @throws java.util.NoSuchElementException
*/
def getSizeAsKb(key: String): Long = {
Utils.byteStringAsKb(get(key))
@@ -337,7 +337,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
/**
* Get a size parameter as Mebibytes; throws a NoSuchElementException if it's not set. If no
* suffix is provided then Mebibytes are assumed.
- * @throws NoSuchElementException
+ * @throws java.util.NoSuchElementException
*/
def getSizeAsMb(key: String): Long = {
Utils.byteStringAsMb(get(key))
@@ -354,7 +354,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
/**
* Get a size parameter as Gibibytes; throws a NoSuchElementException if it's not set. If no
* suffix is provided then Gibibytes are assumed.
- * @throws NoSuchElementException
+ * @throws java.util.NoSuchElementException
*/
def getSizeAsGb(key: String): Long = {
Utils.byteStringAsGb(get(key))
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index a159a170ebc5050e4aad338b84a90e1193a015ac..1cb39a4209a1c0e4a80f2214c19282a22ff53f5e 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -641,7 +641,7 @@ class SparkContext(config: SparkConf) extends Logging {
/**
* Get a local property set in this thread, or null if it is missing. See
- * [[org.apache.spark.SparkContext.setLocalProperty]].
+ * `org.apache.spark.SparkContext.setLocalProperty`.
*/
def getLocalProperty(key: String): String =
Option(localProperties.get).map(_.getProperty(key)).orNull
@@ -659,7 +659,7 @@ class SparkContext(config: SparkConf) extends Logging {
* Application programmers can use this method to group all those jobs together and give a
* group description. Once set, the Spark web UI will associate such jobs with this group.
*
- * The application can also use [[org.apache.spark.SparkContext.cancelJobGroup]] to cancel all
+ * The application can also use `org.apache.spark.SparkContext.cancelJobGroup` to cancel all
* running jobs in this group. For example,
* {{{
* // In the main thread:
@@ -1380,7 +1380,7 @@ class SparkContext(config: SparkConf) extends Logging {
}
/**
- * Create and register a [[CollectionAccumulator]], which starts with empty list and accumulates
+ * Create and register a `CollectionAccumulator`, which starts with empty list and accumulates
* inputs by adding them into the list.
*/
def collectionAccumulator[T]: CollectionAccumulator[T] = {
@@ -1390,7 +1390,7 @@ class SparkContext(config: SparkConf) extends Logging {
}
/**
- * Create and register a [[CollectionAccumulator]], which starts with empty list and accumulates
+ * Create and register a `CollectionAccumulator`, which starts with empty list and accumulates
* inputs by adding them into the list.
*/
def collectionAccumulator[T](name: String): CollectionAccumulator[T] = {
@@ -2039,7 +2039,7 @@ class SparkContext(config: SparkConf) extends Logging {
}
/**
- * Cancel active jobs for the specified group. See [[org.apache.spark.SparkContext.setJobGroup]]
+ * Cancel active jobs for the specified group. See `org.apache.spark.SparkContext.setJobGroup`
* for more information.
*/
def cancelJobGroup(groupId: String) {
@@ -2057,7 +2057,7 @@ class SparkContext(config: SparkConf) extends Logging {
* Cancel a given job if it's scheduled or running.
*
* @param jobId the job ID to cancel
- * @throws InterruptedException if the cancel message cannot be sent
+ * @note Throws `InterruptedException` if the cancel message cannot be sent
*/
def cancelJob(jobId: Int) {
dagScheduler.cancelJob(jobId)
@@ -2067,7 +2067,7 @@ class SparkContext(config: SparkConf) extends Logging {
* Cancel a given stage and all jobs associated with it.
*
* @param stageId the stage ID to cancel
- * @throws InterruptedException if the cancel message cannot be sent
+ * @note Throws `InterruptedException` if the cancel message cannot be sent
*/
def cancelStage(stageId: Int) {
dagScheduler.cancelStage(stageId)
diff --git a/core/src/main/scala/org/apache/spark/TaskContext.scala b/core/src/main/scala/org/apache/spark/TaskContext.scala
index 27abccf5ac2a9203fe4359370f120f57b4031695..0fd777ed12829ceeb51b7a10c0f1b0928e42cd8b 100644
--- a/core/src/main/scala/org/apache/spark/TaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/TaskContext.scala
@@ -164,7 +164,7 @@ abstract class TaskContext extends Serializable {
/**
* Get a local property set upstream in the driver, or null if it is missing. See also
- * [[org.apache.spark.SparkContext.setLocalProperty]].
+ * `org.apache.spark.SparkContext.setLocalProperty`.
*/
def getLocalProperty(key: String): String
@@ -174,7 +174,7 @@ abstract class TaskContext extends Serializable {
/**
* ::DeveloperApi::
* Returns all metrics sources with the given name which are associated with the instance
- * which runs the task. For more information see [[org.apache.spark.metrics.MetricsSystem!]].
+ * which runs the task. For more information see `org.apache.spark.metrics.MetricsSystem`.
*/
@DeveloperApi
def getMetricsSources(sourceName: String): Seq[Source]
diff --git a/core/src/main/scala/org/apache/spark/TaskEndReason.scala b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
index 7ca3c103dbf5b77a90af3e8597804436c916c250..7745387dbcebaab72b12f557ee500fb189229e9c 100644
--- a/core/src/main/scala/org/apache/spark/TaskEndReason.scala
+++ b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
@@ -65,7 +65,7 @@ sealed trait TaskFailedReason extends TaskEndReason {
/**
* :: DeveloperApi ::
- * A [[org.apache.spark.scheduler.ShuffleMapTask]] that completed successfully earlier, but we
+ * A `org.apache.spark.scheduler.ShuffleMapTask` that completed successfully earlier, but we
* lost the executor before the stage completed. This means Spark needs to reschedule the task
* to be re-executed on a different executor.
*/
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index 871b9d1ad575bf86b1920a1de0eb0d906341d6bb..2909191bd6f14b62e558e2e6b80030fc35bb263f 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -186,7 +186,7 @@ private[spark] object TestUtils {
/**
- * A [[SparkListener]] that detects whether spills have occurred in Spark jobs.
+ * A `SparkListener` that detects whether spills have occurred in Spark jobs.
*/
private class SpillListener extends SparkListener {
private val stageIdToTaskMetrics = new mutable.HashMap[Int, ArrayBuffer[TaskMetrics]]
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
index a20d264be5afdf21f90f96e8e87d929a928dd00e..94e26e687c66b86add0195afb2a8a1fbd09b150e 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
@@ -103,7 +103,8 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
* @param withReplacement can elements be sampled multiple times (replaced when sampled out)
* @param fraction expected size of the sample as a fraction of this RDD's size
* without replacement: probability that each element is chosen; fraction must be [0, 1]
- * with replacement: expected number of times each element is chosen; fraction must be >= 0
+ * with replacement: expected number of times each element is chosen; fraction must be greater
+ * than or equal to 0
*
* @note This is NOT guaranteed to provide exactly the fraction of the count
* of the given `RDD`.
@@ -117,7 +118,8 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
* @param withReplacement can elements be sampled multiple times (replaced when sampled out)
* @param fraction expected size of the sample as a fraction of this RDD's size
* without replacement: probability that each element is chosen; fraction must be [0, 1]
- * with replacement: expected number of times each element is chosen; fraction must be >= 0
+ * with replacement: expected number of times each element is chosen; fraction must be greater
+ * than or equal to 0
* @param seed seed for the random number generator
*
* @note This is NOT guaranteed to provide exactly the fraction of the count
@@ -167,7 +169,7 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
* Return an RDD with the elements from `this` that are not in `other`.
*
* Uses `this` partitioner/partition size, because even if `other` is huge, the resulting
- * RDD will be <= us.
+ * RDD will be less than or equal to us.
*/
def subtract(other: JavaRDD[T]): JavaRDD[T] = wrapRDD(rdd.subtract(other))
diff --git a/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala
index f3ab324d5911932211394c767058838584c88fa5..14331dfd0c9870149dec822c06e7fc184808d682 100644
--- a/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala
@@ -152,10 +152,10 @@ class DoubleRDDFunctions(self: RDD[Double]) extends Logging with Serializable {
/**
* Compute a histogram using the provided buckets. The buckets are all open
- * to the right except for the last which is closed
+ * to the right except for the last which is closed.
* e.g. for the array
* [1, 10, 20, 50] the buckets are [1, 10) [10, 20) [20, 50]
- * e.g 1<=x<10 , 10<=x<20, 20<=x<=50
+ * e.g {@code <=x<10, 10<=x<20, 20<=x<=50}
* And on the input of 1 and 50 we would have a histogram of 1, 0, 1
*
* @note If your histogram is evenly spaced (e.g. [0, 10, 20, 30]) this can be switched
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 86351b8c575e5bc7639bce91d39ab3cbbdd7e491..ae4320d4583d6867c252f5b1c6d9def7b44bb31c 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -96,7 +96,7 @@ private[spark] class HadoopPartition(rddId: Int, override val index: Int, s: Inp
* @param minPartitions Minimum number of HadoopRDD partitions (Hadoop Splits) to generate.
*
* @note Instantiating this class directly is not recommended, please use
- * [[org.apache.spark.SparkContext.hadoopRDD()]]
+ * `org.apache.spark.SparkContext.hadoopRDD()`
*/
@DeveloperApi
class HadoopRDD[K, V](
diff --git a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
index 0970b98071675c758e495bc8f5df224676ecaaed..aab46b8954bf71f9dd8f68e4007b92d1d3ab6145 100644
--- a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
@@ -41,7 +41,10 @@ private[spark] class JdbcPartition(idx: Int, val lower: Long, val upper: Long) e
* The RDD takes care of closing the connection.
* @param sql the text of the query.
* The query must contain two ? placeholders for parameters used to partition the results.
- * E.g. "select title, author from books where ? <= id and id <= ?"
+ * For example,
+ * {{{
+ * select title, author from books where ? <= id and id <= ?
+ * }}}
* @param lowerBound the minimum value of the first placeholder
* @param upperBound the maximum value of the second placeholder
* The lower and upper bounds are inclusive.
@@ -151,7 +154,10 @@ object JdbcRDD {
* The RDD takes care of closing the connection.
* @param sql the text of the query.
* The query must contain two ? placeholders for parameters used to partition the results.
- * E.g. "select title, author from books where ? <= id and id <= ?"
+ * For example,
+ * {{{
+ * select title, author from books where ? <= id and id <= ?
+ * }}}
* @param lowerBound the minimum value of the first placeholder
* @param upperBound the maximum value of the second placeholder
* The lower and upper bounds are inclusive.
@@ -191,7 +197,10 @@ object JdbcRDD {
* The RDD takes care of closing the connection.
* @param sql the text of the query.
* The query must contain two ? placeholders for parameters used to partition the results.
- * E.g. "select title, author from books where ? <= id and id <= ?"
+ * For example,
+ * {{{
+ * select title, author from books where ? <= id and id <= ?
+ * }}}
* @param lowerBound the minimum value of the first placeholder
* @param upperBound the maximum value of the second placeholder
* The lower and upper bounds are inclusive.
diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
index a5965f597038d9ad305ff381a727daa569d97ac7..c783e1375283a0a069a0b7919b12c16b2ab67669 100644
--- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
@@ -63,7 +63,7 @@ private[spark] class NewHadoopPartition(
* @param valueClass Class of the value associated with the inputFormatClass.
*
* @note Instantiating this class directly is not recommended, please use
- * [[org.apache.spark.SparkContext.newAPIHadoopRDD()]]
+ * `org.apache.spark.SparkContext.newAPIHadoopRDD()`
*/
@DeveloperApi
class NewHadoopRDD[K, V](
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index 33e695ec5322bb39c6977d6e9baa0a6cabe7651d..aad99e3eb2c5b0df78852cf4545ba8fcf0dab0b7 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -399,9 +399,9 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
* Algorithmic Engineering of a State of The Art Cardinality Estimation Algorithm", available
* <a href="http://dx.doi.org/10.1145/2452376.2452456">here</a>.
*
- * The relative accuracy is approximately `1.054 / sqrt(2^p)`. Setting a nonzero `sp > p`
- * would trigger sparse representation of registers, which may reduce the memory consumption
- * and increase accuracy when the cardinality is small.
+ * The relative accuracy is approximately `1.054 / sqrt(2^p)`. Setting a nonzero (`sp` is
+ * greater than `p`) would trigger sparse representation of registers, which may reduce the
+ * memory consumption and increase accuracy when the cardinality is small.
*
* @param p The precision value for the normal set.
* `p` must be a value between 4 and `sp` if `sp` is not zero (32 max).
@@ -492,8 +492,8 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
* each time the resulting RDD is evaluated.
*
* @note This operation may be very expensive. If you are grouping in order to perform an
- * aggregation (such as a sum or average) over each key, using [[PairRDDFunctions.aggregateByKey]]
- * or [[PairRDDFunctions.reduceByKey]] will provide much better performance.
+ * aggregation (such as a sum or average) over each key, using `PairRDDFunctions.aggregateByKey`
+ * or `PairRDDFunctions.reduceByKey` will provide much better performance.
*
* @note As currently implemented, groupByKey must be able to hold all the key-value pairs for any
* key in memory. If a key has too many values, it can result in an [[OutOfMemoryError]].
@@ -516,8 +516,8 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
* each group is not guaranteed, and may even differ each time the resulting RDD is evaluated.
*
* @note This operation may be very expensive. If you are grouping in order to perform an
- * aggregation (such as a sum or average) over each key, using [[PairRDDFunctions.aggregateByKey]]
- * or [[PairRDDFunctions.reduceByKey]] will provide much better performance.
+ * aggregation (such as a sum or average) over each key, using `PairRDDFunctions.aggregateByKey`
+ * or `PairRDDFunctions.reduceByKey` will provide much better performance.
*
* @note As currently implemented, groupByKey must be able to hold all the key-value pairs for any
* key in memory. If a key has too many values, it can result in an [[OutOfMemoryError]].
@@ -637,8 +637,8 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
* evaluated.
*
* @note This operation may be very expensive. If you are grouping in order to perform an
- * aggregation (such as a sum or average) over each key, using [[PairRDDFunctions.aggregateByKey]]
- * or [[PairRDDFunctions.reduceByKey]] will provide much better performance.
+ * aggregation (such as a sum or average) over each key, using `PairRDDFunctions.aggregateByKey`
+ * or `PairRDDFunctions.reduceByKey` will provide much better performance.
*/
def groupByKey(): RDD[(K, Iterable[V])] = self.withScope {
groupByKey(defaultPartitioner(self))
@@ -908,7 +908,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
* Return an RDD with the pairs from `this` whose keys are not in `other`.
*
* Uses `this` partitioner/partition size, because even if `other` is huge, the resulting
- * RDD will be <= us.
+ * RDD will be less than or equal to us.
*/
def subtractByKey[W: ClassTag](other: RDD[(K, W)]): RDD[(K, V)] = self.withScope {
subtractByKey(other, self.partitioner.getOrElse(new HashPartitioner(self.partitions.length)))
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 8e673447581cfe16f2f4def8fef9462fe03ebeaa..f723fcb837f8819f72febe197613c5cb13be66d5 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -469,7 +469,8 @@ abstract class RDD[T: ClassTag](
* @param withReplacement can elements be sampled multiple times (replaced when sampled out)
* @param fraction expected size of the sample as a fraction of this RDD's size
* without replacement: probability that each element is chosen; fraction must be [0, 1]
- * with replacement: expected number of times each element is chosen; fraction must be >= 0
+ * with replacement: expected number of times each element is chosen; fraction must be greater
+ * than or equal to 0
* @param seed seed for the random number generator
*
* @note This is NOT guaranteed to provide exactly the fraction of the count
@@ -750,8 +751,10 @@ abstract class RDD[T: ClassTag](
* print line function (like out.println()) as the 2nd parameter.
* An example of pipe the RDD data of groupBy() in a streaming way,
* instead of constructing a huge String to concat all the elements:
- * def printRDDElement(record:(String, Seq[String]), f:String=>Unit) =
- * for (e <- record._2) {f(e)}
+ * {{{
+ * def printRDDElement(record:(String, Seq[String]), f:String=>Unit) =
+ * for (e <- record._2) {f(e)}
+ * }}}
* @param separateWorkingDir Use separate working directories for each task.
* @param bufferSize Buffer size for the stdin writer for the piped process.
* @param encoding Char encoding used for interacting (via stdin, stdout and stderr) with
@@ -1184,8 +1187,13 @@ abstract class RDD[T: ClassTag](
*
* @note This method should only be used if the resulting map is expected to be small, as
* the whole thing is loaded into the driver's memory.
- * To handle very large results, consider using rdd.map(x => (x, 1L)).reduceByKey(_ + _), which
- * returns an RDD[T, Long] instead of a map.
+ * To handle very large results, consider using
+ *
+ * {{{
+ * rdd.map(x => (x, 1L)).reduceByKey(_ + _)
+ * }}}
+ *
+ * , which returns an RDD[T, Long] instead of a map.
*/
def countByValue()(implicit ord: Ordering[T] = null): Map[T, Long] = withScope {
map(value => (value, null)).countByKey()
@@ -1223,9 +1231,9 @@ abstract class RDD[T: ClassTag](
* Algorithmic Engineering of a State of The Art Cardinality Estimation Algorithm", available
* <a href="http://dx.doi.org/10.1145/2452376.2452456">here</a>.
*
- * The relative accuracy is approximately `1.054 / sqrt(2^p)`. Setting a nonzero `sp > p`
- * would trigger sparse representation of registers, which may reduce the memory consumption
- * and increase accuracy when the cardinality is small.
+ * The relative accuracy is approximately `1.054 / sqrt(2^p)`. Setting a nonzero (`sp` is greater
+ * than `p`) would trigger sparse representation of registers, which may reduce the memory
+ * consumption and increase accuracy when the cardinality is small.
*
* @param p The precision value for the normal set.
* `p` must be a value between 4 and `sp` if `sp` is not zero (32 max).
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDCheckpointData.scala b/core/src/main/scala/org/apache/spark/rdd/RDDCheckpointData.scala
index 1070bb96b25247415c4111bc84073396adf393b6..6c552d4d12515a7e549aee1138ce67f5a877297a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDCheckpointData.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDCheckpointData.scala
@@ -23,7 +23,8 @@ import org.apache.spark.Partition
/**
* Enumeration to manage state transitions of an RDD through checkpointing
- * [ Initialized --> checkpointing in progress --> checkpointed ].
+ *
+ * [ Initialized --{@literal >} checkpointing in progress --{@literal >} checkpointed ]
*/
private[spark] object CheckpointState extends Enumeration {
type CheckpointState = Value
diff --git a/core/src/main/scala/org/apache/spark/rdd/coalesce-public.scala b/core/src/main/scala/org/apache/spark/rdd/coalesce-public.scala
index d8a80aa5aeb15e8683a851a230a31bf62be911bc..e00bc22aba44d88190a4f867bcf209aced4df7bb 100644
--- a/core/src/main/scala/org/apache/spark/rdd/coalesce-public.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/coalesce-public.scala
@@ -35,14 +35,14 @@ trait PartitionCoalescer {
* @param maxPartitions the maximum number of partitions to have after coalescing
* @param parent the parent RDD whose partitions to coalesce
* @return an array of [[PartitionGroup]]s, where each element is itself an array of
- * [[Partition]]s and represents a partition after coalescing is performed.
+ * `Partition`s and represents a partition after coalescing is performed.
*/
def coalesce(maxPartitions: Int, parent: RDD[_]): Array[PartitionGroup]
}
/**
* ::DeveloperApi::
- * A group of [[Partition]]s
+ * A group of `Partition`s
* @param prefLoc preferred location for the partition group
*/
@DeveloperApi
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/RpcEndpointVerifier.scala b/core/src/main/scala/org/apache/spark/rpc/netty/RpcEndpointVerifier.scala
index 99f20da2d66aa282a68c5a1176742e5a0480341a..430dcc50ba71100d31403bda80f67ee329795497 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/RpcEndpointVerifier.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/RpcEndpointVerifier.scala
@@ -20,7 +20,7 @@ package org.apache.spark.rpc.netty
import org.apache.spark.rpc.{RpcCallContext, RpcEndpoint, RpcEnv}
/**
- * An [[RpcEndpoint]] for remote [[RpcEnv]]s to query if an [[RpcEndpoint]] exists.
+ * An [[RpcEndpoint]] for remote [[RpcEnv]]s to query if an `RpcEndpoint` exists.
*
* This is used when setting up a remote endpoint reference.
*/
@@ -35,6 +35,6 @@ private[netty] class RpcEndpointVerifier(override val rpcEnv: RpcEnv, dispatcher
private[netty] object RpcEndpointVerifier {
val NAME = "endpoint-verifier"
- /** A message used to ask the remote [[RpcEndpointVerifier]] if an [[RpcEndpoint]] exists. */
+ /** A message used to ask the remote [[RpcEndpointVerifier]] if an `RpcEndpoint` exists. */
case class CheckExistence(name: String)
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala
index a6b032cc0084ce559473f7954132295606553d0b..66ab9a52b778118c0e82f307a7b3fea4b179e50c 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala
@@ -153,7 +153,7 @@ object InputFormatInfo {
a) For each host, count number of splits hosted on that host.
b) Decrement the currently allocated containers on that host.
- c) Compute rack info for each host and update rack -> count map based on (b).
+ c) Compute rack info for each host and update rack to count map based on (b).
d) Allocate nodes based on (c)
e) On the allocation result, ensure that we don't allocate "too many" jobs on a single node
(even if data locality on that is very high) : this is to prevent fragility of job if a
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
index 1e7c63af2e797f108682bf6c7217ba8f13fe2833..d19353f2a9930a0355ca4c1a3499302451c5455f 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
@@ -42,7 +42,7 @@ import org.apache.spark.rdd.RDD
* @param outputId index of the task in this job (a job can launch tasks on only a subset of the
* input RDD's partitions).
* @param localProperties copy of thread-local properties set by the user on the driver side.
- * @param metrics a [[TaskMetrics]] that is created at driver side and sent to executor side.
+ * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side.
*
* The parameters below are optional:
* @param jobId id of the job this task belongs to
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
index 66d6790e168f258b8540535eb8ca752aed56d7da..31011de85bf7e4dc523edcabf4c97f93ae648aa9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
@@ -42,7 +42,7 @@ import org.apache.spark.shuffle.ShuffleWriter
* the type should be (RDD[_], ShuffleDependency[_, _, _]).
* @param partition partition of the RDD this task is associated with
* @param locs preferred task execution locations for locality scheduling
- * @param metrics a [[TaskMetrics]] that is created at driver side and sent to executor side.
+ * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side.
* @param localProperties copy of thread-local properties set by the user on the driver side.
*
* The parameters below are optional:
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
index d39651a722325ab3f77aead9e21dabcb3b90ecc5..1554200aeaf64930298b6cbe09a1f038c7b70fe1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
@@ -46,7 +46,7 @@ import org.apache.spark.util._
* @param stageId id of the stage this task belongs to
* @param stageAttemptId attempt id of the stage this task belongs to
* @param partitionId index of the number in the RDD
- * @param metrics a [[TaskMetrics]] that is created at driver side and sent to executor side.
+ * @param metrics a `TaskMetrics` that is created at driver side and sent to executor side.
* @param localProperties copy of thread-local properties set by the user on the driver side.
*
* The parameters below are optional:
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala
index 1c7c81c488c3a39bf9bcf18d5059e2618630b699..45c742cbff5e7d893727475d28445bae0741cf6b 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala
@@ -23,7 +23,7 @@ import org.apache.spark.util.SerializableBuffer
/**
* Description of a task that gets passed onto executors to be executed, usually created by
- * [[TaskSetManager.resourceOffer]].
+ * `TaskSetManager.resourceOffer`.
*/
private[spark] class TaskDescription(
val taskId: Long,
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala
index 6bded927005044b3b60d0aff547da574e77380e4..d71acbb4cf77166e8fd1da5a3d486a4734541ee8 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMessages.scala
@@ -43,7 +43,7 @@ private[spark] object BlockManagerMessages {
extends ToBlockManagerSlave
/**
- * Driver -> Executor message to trigger a thread dump.
+ * Driver to Executor message to trigger a thread dump.
*/
case object TriggerThreadDump extends ToBlockManagerSlave
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index 4dc2f362329a0665e74237a5f651250c12a2694f..269c12d6da4447314f867c364326dc6b50c292da 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -247,7 +247,7 @@ final class ShuffleBlockFetcherIterator(
/**
* Fetch the local blocks while we are fetching remote blocks. This is ok because
- * [[ManagedBuffer]]'s memory is allocated lazily when we create the input stream, so all we
+ * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we
* track in-memory are the ManagedBuffer references themselves.
*/
private[this] def fetchLocalBlocks() {
@@ -423,7 +423,7 @@ object ShuffleBlockFetcherIterator {
* @param address BlockManager that the block was fetched from.
* @param size estimated size of the block, used to calculate bytesInFlight.
* Note that this is NOT the exact bytes.
- * @param buf [[ManagedBuffer]] for the content.
+ * @param buf `ManagedBuffer` for the content.
* @param isNetworkReqDone Is this the last network request for this host in this fetch request.
*/
private[storage] case class SuccessFetchResult(
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index dbeb970c81dfeed51f1c86dc3558359e6e1d7134..d161843dd22304b837fa7d30e5665c06371a3151 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -422,7 +422,7 @@ private[spark] object UIUtils extends Logging {
* the whole string will rendered as a simple escaped text.
*
* Note: In terms of security, only anchor tags with root relative links are supported. So any
- * attempts to embed links outside Spark UI, or other tags like <script> will cause in
+ * attempts to embed links outside Spark UI, or other tags like {@code <script>} will cause in
* the whole description to be treated as plain text.
*
* @param desc the original job or stage description string, which may contain html tags.
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 53770503a309aa4fa675f157edcc653f82a75d4f..91f5606127f427cdf9f9c06d462665868c96c752 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1674,7 +1674,7 @@ private[spark] object Utils extends Logging {
/**
* NaN-safe version of `java.lang.Double.compare()` which allows NaN values to be compared
- * according to semantics where NaN == NaN and NaN > any non-NaN double.
+ * according to semantics where NaN == NaN and NaN is greater than any non-NaN double.
*/
def nanSafeCompareDoubles(x: Double, y: Double): Int = {
val xIsNan: Boolean = java.lang.Double.isNaN(x)
@@ -1688,7 +1688,7 @@ private[spark] object Utils extends Logging {
/**
* NaN-safe version of `java.lang.Float.compare()` which allows NaN values to be compared
- * according to semantics where NaN == NaN and NaN > any non-NaN float.
+ * according to semantics where NaN == NaN and NaN is greater than any non-NaN float.
*/
def nanSafeCompareFloats(x: Float, y: Float): Int = {
val xIsNan: Boolean = java.lang.Float.isNaN(x)
@@ -2354,8 +2354,9 @@ private[spark] object Utils extends Logging {
* A spark url (`spark://host:port`) is a special URI that its scheme is `spark` and only contains
* host and port.
*
- * @note Throws `SparkException` if sparkUrl is invalid.
+ * @throws org.apache.spark.SparkException if sparkUrl is invalid.
*/
+ @throws(classOf[SparkException])
def extractHostPortFromSparkUrl(sparkUrl: String): (String, Int) = {
try {
val uri = new java.net.URI(sparkUrl)
diff --git a/core/src/main/scala/org/apache/spark/util/random/SamplingUtils.scala b/core/src/main/scala/org/apache/spark/util/random/SamplingUtils.scala
index f98932a4701659619ed1ab616e4cdb91e800ee48..297524c943e1fe49bde58bbb5c38f0f5b3624b2d 100644
--- a/core/src/main/scala/org/apache/spark/util/random/SamplingUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/random/SamplingUtils.scala
@@ -67,17 +67,19 @@ private[spark] object SamplingUtils {
}
/**
- * Returns a sampling rate that guarantees a sample of size >= sampleSizeLowerBound 99.99% of
- * the time.
+ * Returns a sampling rate that guarantees a sample of size greater than or equal to
+ * sampleSizeLowerBound 99.99% of the time.
*
* How the sampling rate is determined:
+ *
* Let p = num / total, where num is the sample size and total is the total number of
- * datapoints in the RDD. We're trying to compute q > p such that
+ * datapoints in the RDD. We're trying to compute q {@literal >} p such that
* - when sampling with replacement, we're drawing each datapoint with prob_i ~ Pois(q),
- * where we want to guarantee Pr[s < num] < 0.0001 for s = sum(prob_i for i from 0 to total),
- * i.e. the failure rate of not having a sufficiently large sample < 0.0001.
+ * where we want to guarantee
+ * Pr[s {@literal <} num] {@literal <} 0.0001 for s = sum(prob_i for i from 0 to total),
+ * i.e. the failure rate of not having a sufficiently large sample {@literal <} 0.0001.
* Setting q = p + 5 * sqrt(p/total) is sufficient to guarantee 0.9999 success rate for
- * num > 12, but we need a slightly larger q (9 empirically determined).
+ * num {@literal >} 12, but we need a slightly larger q (9 empirically determined).
* - when sampling without replacement, we're drawing each datapoint with prob_i
* ~ Binomial(total, fraction) and our choice of q guarantees 1-delta, or 0.9999 success
* rate, where success rate is defined the same as in sampling with replacement.
@@ -108,14 +110,14 @@ private[spark] object SamplingUtils {
private[spark] object PoissonBounds {
/**
- * Returns a lambda such that Pr[X > s] is very small, where X ~ Pois(lambda).
+ * Returns a lambda such that Pr[X {@literal >} s] is very small, where X ~ Pois(lambda).
*/
def getLowerBound(s: Double): Double = {
math.max(s - numStd(s) * math.sqrt(s), 1e-15)
}
/**
- * Returns a lambda such that Pr[X < s] is very small, where X ~ Pois(lambda).
+ * Returns a lambda such that Pr[X {@literal <} s] is very small, where X ~ Pois(lambda).
*
* @param s sample size
*/
diff --git a/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala b/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala
index 67822749112c6bfffde0e93982038600b08c597a..ce46fc8f201bed78136079699539a61a9bc8971e 100644
--- a/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala
@@ -35,13 +35,14 @@ import org.apache.spark.rdd.RDD
* high probability. This is achieved by maintaining a waitlist of size O(log(s)), where s is the
* desired sample size for each stratum.
*
- * Like in simple random sampling, we generate a random value for each item from the
- * uniform distribution [0.0, 1.0]. All items with values <= min(values of items in the waitlist)
- * are accepted into the sample instantly. The threshold for instant accept is designed so that
- * s - numAccepted = O(sqrt(s)), where s is again the desired sample size. Thus, by maintaining a
- * waitlist size = O(sqrt(s)), we will be able to create a sample of the exact size s by adding
- * a portion of the waitlist to the set of items that are instantly accepted. The exact threshold
- * is computed by sorting the values in the waitlist and picking the value at (s - numAccepted).
+ * Like in simple random sampling, we generate a random value for each item from the uniform
+ * distribution [0.0, 1.0]. All items with values less than or equal to min(values of items in the
+ * waitlist) are accepted into the sample instantly. The threshold for instant accept is designed
+ * so that s - numAccepted = O(sqrt(s)), where s is again the desired sample size. Thus, by
+ * maintaining a waitlist size = O(sqrt(s)), we will be able to create a sample of the exact size
+ * s by adding a portion of the waitlist to the set of items that are instantly accepted. The exact
+ * threshold is computed by sorting the values in the waitlist and picking the value at
+ * (s - numAccepted).
*
* Note that since we use the same seed for the RNG when computing the thresholds and the actual
* sample, our computed thresholds are guaranteed to produce the desired sample size.
@@ -160,12 +161,20 @@ private[spark] object StratifiedSamplingUtils extends Logging {
*
* To do so, we compute sampleSize = math.ceil(size * samplingRate) for each stratum and compare
* it to the number of items that were accepted instantly and the number of items in the waitlist
- * for that stratum. Most of the time, numAccepted <= sampleSize <= (numAccepted + numWaitlisted),
+ * for that stratum.
+ *
+ * Most of the time,
+ * {{{
+ * numAccepted <= sampleSize <= (numAccepted + numWaitlisted)
+ * }}}
* which means we need to sort the elements in the waitlist by their associated values in order
- * to find the value T s.t. |{elements in the stratum whose associated values <= T}| = sampleSize.
- * Note that all elements in the waitlist have values >= bound for instant accept, so a T value
- * in the waitlist range would allow all elements that were instantly accepted on the first pass
- * to be included in the sample.
+ * to find the value T s.t.
+ * {{{
+ * |{elements in the stratum whose associated values <= T}| = sampleSize
+ * }}}.
+ * Note that all elements in the waitlist have values greater than or equal to bound for instant
+ * accept, so a T value in the waitlist range would allow all elements that were instantly
+ * accepted on the first pass to be included in the sample.
*/
def computeThresholdByKey[K](finalResult: Map[K, AcceptanceResult],
fractions: Map[K, Double]): Map[K, Double] = {
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala
index 54565840fa665e77fffa8a6c366f69566e905e7d..d84e289272c6221053a0f45a22c38b7ce688bf28 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala
@@ -36,7 +36,7 @@ import org.apache.spark.streaming.flume.sink._
import org.apache.spark.streaming.receiver.Receiver
/**
- * A [[ReceiverInputDStream]] that can be used to read data from several Flume agents running
+ * A `ReceiverInputDStream` that can be used to read data from several Flume agents running
* [[org.apache.spark.streaming.flume.sink.SparkSink]]s.
* @param _ssc Streaming context that will execute this input stream
* @param addresses List of addresses at which SparkSinks are listening
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala
index 35acb7b09f12b4d7c93b2013778bcb1f89f62562..e0e44d4440272bda84883b30ebf171c0e1d2d451 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaCluster.scala
@@ -231,7 +231,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable {
// this 0 here indicates api version, in this case the original ZK backed api.
private def defaultConsumerApiVersion: Short = 0
- /** Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version. */
+ /**
+ * Requires Kafka 0.8.1.1 or later.
+ * Defaults to the original ZooKeeper backed API version.
+ */
def getConsumerOffsets(
groupId: String,
topicAndPartitions: Set[TopicAndPartition]
@@ -250,7 +253,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable {
}
}
- /** Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version. */
+ /**
+ * Requires Kafka 0.8.1.1 or later.
+ * Defaults to the original ZooKeeper backed API version.
+ */
def getConsumerOffsetMetadata(
groupId: String,
topicAndPartitions: Set[TopicAndPartition]
@@ -287,7 +293,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable {
Left(errs)
}
- /** Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version. */
+ /**
+ * Requires Kafka 0.8.1.1 or later.
+ * Defaults to the original ZooKeeper backed API version.
+ */
def setConsumerOffsets(
groupId: String,
offsets: Map[TopicAndPartition, Long]
@@ -305,7 +314,10 @@ class KafkaCluster(val kafkaParams: Map[String, String]) extends Serializable {
setConsumerOffsetMetadata(groupId, meta, consumerApiVersion)
}
- /** Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version. */
+ /**
+ * Requires Kafka 0.8.1.1 or later.
+ * Defaults to the original ZooKeeper backed API version.
+ */
def setConsumerOffsetMetadata(
groupId: String,
metadata: Map[TopicAndPartition, OffsetAndMetadata]
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaInputDStream.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaInputDStream.scala
index 3713bda41b8eeba228184983986b5f39eed82462..7ff3a98ca52cd265956ba8b4f928e614ff586338 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaInputDStream.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaInputDStream.scala
@@ -38,7 +38,7 @@ import org.apache.spark.util.ThreadUtils
*
* @param kafkaParams Map of kafka configuration parameters.
* See: http://kafka.apache.org/configuration.html
- * @param topics Map of (topic_name -> numPartitions) to consume. Each partition is consumed
+ * @param topics Map of (topic_name to numPartitions) to consume. Each partition is consumed
* in its own thread.
* @param storageLevel RDD storage level.
*/
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
index 56f0cb0b166a2941e97aae4fe574dc8407b5cec5..d5aef8184fc87c4dd1e446199fa3a29c246fdd9b 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
@@ -47,7 +47,7 @@ object KafkaUtils {
* @param ssc StreamingContext object
* @param zkQuorum Zookeeper quorum (hostname:port,hostname:port,..)
* @param groupId The group id for this consumer
- * @param topics Map of (topic_name -> numPartitions) to consume. Each partition is consumed
+ * @param topics Map of (topic_name to numPartitions) to consume. Each partition is consumed
* in its own thread
* @param storageLevel Storage level to use for storing the received objects
* (default: StorageLevel.MEMORY_AND_DISK_SER_2)
@@ -72,7 +72,7 @@ object KafkaUtils {
* @param ssc StreamingContext object
* @param kafkaParams Map of kafka configuration parameters,
* see http://kafka.apache.org/08/configuration.html
- * @param topics Map of (topic_name -> numPartitions) to consume. Each partition is consumed
+ * @param topics Map of (topic_name to numPartitions) to consume. Each partition is consumed
* in its own thread.
* @param storageLevel Storage level to use for storing the received objects
* @tparam K type of Kafka message key
@@ -97,7 +97,7 @@ object KafkaUtils {
* @param jssc JavaStreamingContext object
* @param zkQuorum Zookeeper quorum (hostname:port,hostname:port,..)
* @param groupId The group id for this consumer
- * @param topics Map of (topic_name -> numPartitions) to consume. Each partition is consumed
+ * @param topics Map of (topic_name to numPartitions) to consume. Each partition is consumed
* in its own thread
* @return DStream of (Kafka message key, Kafka message value)
*/
@@ -115,7 +115,7 @@ object KafkaUtils {
* @param jssc JavaStreamingContext object
* @param zkQuorum Zookeeper quorum (hostname:port,hostname:port,..).
* @param groupId The group id for this consumer.
- * @param topics Map of (topic_name -> numPartitions) to consume. Each partition is consumed
+ * @param topics Map of (topic_name to numPartitions) to consume. Each partition is consumed
* in its own thread.
* @param storageLevel RDD storage level.
* @return DStream of (Kafka message key, Kafka message value)
@@ -140,7 +140,7 @@ object KafkaUtils {
* @param valueDecoderClass Type of kafka value decoder
* @param kafkaParams Map of kafka configuration parameters,
* see http://kafka.apache.org/08/configuration.html
- * @param topics Map of (topic_name -> numPartitions) to consume. Each partition is consumed
+ * @param topics Map of (topic_name to numPartitions) to consume. Each partition is consumed
* in its own thread
* @param storageLevel RDD storage level.
* @tparam K type of Kafka message key
@@ -396,7 +396,7 @@ object KafkaUtils {
* You can access the offsets used in each batch from the generated RDDs (see
* [[org.apache.spark.streaming.kafka.HasOffsetRanges]]).
* - Failure Recovery: To recover from driver failures, you have to enable checkpointing
- * in the [[StreamingContext]]. The information on consumed offset can be
+ * in the `StreamingContext`. The information on consumed offset can be
* recovered from the checkpoint. See the programming guide for details (constraints, etc.).
* - End-to-end semantics: This stream ensures that every records is effectively received and
* transformed exactly once, but gives no guarantees on whether the transformed data are
@@ -448,7 +448,7 @@ object KafkaUtils {
* You can access the offsets used in each batch from the generated RDDs (see
* [[org.apache.spark.streaming.kafka.HasOffsetRanges]]).
* - Failure Recovery: To recover from driver failures, you have to enable checkpointing
- * in the [[StreamingContext]]. The information on consumed offset can be
+ * in the `StreamingContext`. The information on consumed offset can be
* recovered from the checkpoint. See the programming guide for details (constraints, etc.).
* - End-to-end semantics: This stream ensures that every records is effectively received and
* transformed exactly once, but gives no guarantees on whether the transformed data are
@@ -499,7 +499,7 @@ object KafkaUtils {
* You can access the offsets used in each batch from the generated RDDs (see
* [[org.apache.spark.streaming.kafka.HasOffsetRanges]]).
* - Failure Recovery: To recover from driver failures, you have to enable checkpointing
- * in the [[StreamingContext]]. The information on consumed offset can be
+ * in the `StreamingContext`. The information on consumed offset can be
* recovered from the checkpoint. See the programming guide for details (constraints, etc.).
* - End-to-end semantics: This stream ensures that every records is effectively received and
* transformed exactly once, but gives no guarantees on whether the transformed data are
@@ -565,7 +565,7 @@ object KafkaUtils {
* You can access the offsets used in each batch from the generated RDDs (see
* [[org.apache.spark.streaming.kafka.HasOffsetRanges]]).
* - Failure Recovery: To recover from driver failures, you have to enable checkpointing
- * in the [[StreamingContext]]. The information on consumed offset can be
+ * in the `StreamingContext`. The information on consumed offset can be
* recovered from the checkpoint. See the programming guide for details (constraints, etc.).
* - End-to-end semantics: This stream ensures that every records is effectively received and
* transformed exactly once, but gives no guarantees on whether the transformed data are
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala
index d9b856e4697a0fd8af443ed416ffcee8d3e47dba..10d364f987405be6ef2d1ffbcde0410405e0dce0 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/OffsetRange.scala
@@ -22,7 +22,7 @@ import kafka.common.TopicAndPartition
/**
* Represents any object that has a collection of [[OffsetRange]]s. This can be used to access the
* offset ranges in RDDs generated by the direct Kafka DStream (see
- * [[KafkaUtils.createDirectStream()]]).
+ * `KafkaUtils.createDirectStream()`).
* {{{
* KafkaUtils.createDirectStream(...).foreachRDD { rdd =>
* val offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala b/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
index add21f41ea3ba8d0fa888070bb0f3d849f7eb719..f665727ef90db913d29529483456d5af3e589a79 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
@@ -32,7 +32,7 @@ object GraphLoader extends Logging {
* id and a target id. Skips lines that begin with `#`.
*
* If desired the edges can be automatically oriented in the positive
- * direction (source Id < target Id) by setting `canonicalOrientation` to
+ * direction (source Id is less than target Id) by setting `canonicalOrientation` to
* true.
*
* @example Loads a file in the following format:
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala
index 8d608c99b1a1d752dd51d9b70d01c621b292e918..8da46db98be815c0e65429fe3c5342f48936590e 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBase.scala
@@ -57,7 +57,7 @@ private[graphx] object VertexPartitionBase {
* concrete implementation. [[VertexPartitionBaseOps]] provides a variety of operations for
* VertexPartitionBase and subclasses that provide implicit evidence of membership in the
* `VertexPartitionBaseOpsConstructor` typeclass (for example,
- * [[VertexPartition.VertexPartitionOpsConstructor]]).
+ * `VertexPartition.VertexPartitionOpsConstructor`).
*/
private[graphx] abstract class VertexPartitionBase[@specialized(Long, Int, Double) VD: ClassTag]
extends Serializable {
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala
index 43594573cf0136737f5c0dc4e01544a377e30d84..a8ed59b09bbb7b1d3ce4dba16398477959ed854b 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexPartitionBaseOps.scala
@@ -29,7 +29,7 @@ import org.apache.spark.util.collection.BitSet
/**
* A class containing additional operations for subclasses of VertexPartitionBase that provide
* implicit evidence of membership in the `VertexPartitionBaseOpsConstructor` typeclass (for
- * example, [[VertexPartition.VertexPartitionOpsConstructor]]).
+ * example, `VertexPartition.VertexPartitionOpsConstructor`).
*/
private[graphx] abstract class VertexPartitionBaseOps
[VD: ClassTag, Self[X] <: VertexPartitionBase[X]: VertexPartitionBaseOpsConstructor]
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
index 21b22968a1a69ed0ad871e68e90cb483e68cb753..2715137d19ebc02cae530cbf96e9c65637c2db1d 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/TriangleCount.scala
@@ -36,7 +36,7 @@ import org.apache.spark.graphx._
* self cycles and canonicalizes the graph to ensure that the following conditions hold:
* <ul>
* <li> There are no self edges</li>
- * <li> All edges are oriented src > dst</li>
+ * <li> All edges are oriented (src is greater than dst)</li>
* <li> There are no duplicate edges</li>
* </ul>
* However, the canonicalization procedure is costly as it requires repartitioning the graph.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 41b84f481633cf6fa1f3e91f63a0ec589c60d342..ec582266e6a47031bfae2000244a3eb3758a5229 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -56,8 +56,8 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
/**
* Set threshold in binary classification, in range [0, 1].
*
- * If the estimated probability of class label 1 is > threshold, then predict 1, else 0.
- * A high threshold encourages the model to predict 0 more often;
+ * If the estimated probability of class label 1 is greater than threshold, then predict 1,
+ * else 0. A high threshold encourages the model to predict 0 more often;
* a low threshold encourages the model to predict 1 more often.
*
* Note: Calling this with threshold p is equivalent to calling `setThresholds(Array(1-p, p))`.
@@ -123,7 +123,7 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
/**
* Set thresholds in multiclass (or binary) classification to adjust the probability of
* predicting each class. Array must have length equal to the number of classes,
- * with values > 0, excepting that at most one value may be 0.
+ * with values greater than 0, excepting that at most one value may be 0.
* The class with largest value p/t is predicted, where p is the original probability of that
* class and t is the class's threshold.
*
@@ -210,8 +210,9 @@ class LogisticRegression @Since("1.2.0") (
/**
* Set the ElasticNet mixing parameter.
- * For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.
- * For 0 < alpha < 1, the penalty is a combination of L1 and L2.
+ * For alpha = 0, the penalty is an L2 penalty.
+ * For alpha = 1, it is an L1 penalty.
+ * For alpha in (0,1), the penalty is a combination of L1 and L2.
* Default is 0.0 which is an L2 penalty.
*
* @group setParam
@@ -298,7 +299,7 @@ class LogisticRegression @Since("1.2.0") (
override def getThresholds: Array[Double] = super.getThresholds
/**
- * Suggested depth for treeAggregate (>= 2).
+ * Suggested depth for treeAggregate (greater than or equal to 2).
* If the dimensions of features or the number of partitions are large,
* this param could be adjusted to a larger size.
* Default is 2.
@@ -1517,7 +1518,7 @@ private class LogisticAggregator(
}
/**
- * When maxMargin > 0, the original formula could cause overflow.
+ * When maxMargin is greater than 0, the original formula could cause overflow.
* We address this by subtracting maxMargin from all the margins, so it's guaranteed
* that all of the new margins will be smaller than zero to prevent arithmetic overflow.
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index c7a170ddc7351b0fb0068b0b4a0caa7dc0d497cd..e58df6ba9108ab53480679efe72dbdbe6c57555d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -55,8 +55,8 @@ private[clustering] trait BisectingKMeansParams extends Params
def getK: Int = $(k)
/**
- * The minimum number of points (if >= 1.0) or the minimum proportion
- * of points (if < 1.0) of a divisible cluster (default: 1.0).
+ * The minimum number of points (if greater than or equal to 1.0) or the minimum proportion
+ * of points (if less than 1.0) of a divisible cluster (default: 1.0).
* @group expertParam
*/
@Since("2.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 74109344aac0853b9b60371dfb43ba63e9f450d3..c764c3aa32a4c3b38ac1d03438b35fb775cddee7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -44,7 +44,7 @@ private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter w
with HasSeed with HasPredictionCol with HasProbabilityCol with HasTol {
/**
- * Number of independent Gaussians in the mixture model. Must be > 1. Default: 2.
+ * Number of independent Gaussians in the mixture model. Must be greater than 1. Default: 2.
* @group param
*/
@Since("2.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 6032ab3db93503d8e82b2d9241cd007bd4426c8f..cd403d842b694c7ab8a699d767c6b397eb425720 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -78,11 +78,11 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
* - EM
* - Currently only supports symmetric distributions, so all values in the vector should be
* the same.
- * - Values should be > 1.0
+ * - Values should be greater than 1.0
* - default = uniformly (50 / k) + 1, where 50/k is common in LDA libraries and +1 follows
* from Asuncion et al. (2009), who recommend a +1 adjustment for EM.
* - Online
- * - Values should be >= 0
+ * - Values should be greater than or equal to 0
* - default = uniformly (1.0 / k), following the implementation from
* <a href="https://github.com/Blei-Lab/onlineldavb">here</a>.
*
@@ -120,11 +120,11 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
*
* Optimizer-specific parameter settings:
* - EM
- * - Value should be > 1.0
+ * - Value should be greater than 1.0
* - default = 0.1 + 1, where 0.1 gives a small amount of smoothing and +1 follows
* Asuncion et al. (2009), who recommend a +1 adjustment for EM.
* - Online
- * - Value should be >= 0
+ * - Value should be greater than or equal to 0
* - default = (1.0 / k), following the implementation from
* <a href="https://github.com/Blei-Lab/onlineldavb">here</a>.
*
@@ -247,7 +247,7 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
*
* Note that this should be adjusted in synch with `LDA.maxIter`
* so the entire corpus is used. Specifically, set both so that
- * maxIterations * miniBatchFraction >= 1.
+ * maxIterations * miniBatchFraction greater than or equal to 1.
*
* Note: This is the same as the `miniBatchFraction` parameter in
* [[org.apache.spark.mllib.clustering.OnlineLDAOptimizer]].
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
index 1143f0f565ebd1d941151e7ff53cddb476c40d65..260159f8b7ac4e5c44abd840e24f6a9c9114cd27 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
@@ -44,7 +44,7 @@ final class Bucketizer @Since("1.4.0") (@Since("1.4.0") override val uid: String
/**
* Parameter for mapping continuous features into buckets. With n+1 splits, there are n buckets.
* A bucket defined by splits x,y holds values in the range [x,y) except the last bucket, which
- * also includes y. Splits should be of length >= 3 and strictly increasing.
+ * also includes y. Splits should be of length greater than or equal to 3 and strictly increasing.
* Values at -inf, inf must be explicitly provided to cover all Double values;
* otherwise, values outside the splits specified will be treated as errors.
*
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
index 6299f74a6bf96872cf2ad10d525df006ef7a15ab..1ebe29703bc47290533df59ac016e27fb29f0e94 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
@@ -53,8 +53,9 @@ private[feature] trait CountVectorizerParams extends Params with HasInputCol wit
/**
* Specifies the minimum number of different documents a term must appear in to be included
* in the vocabulary.
- * If this is an integer >= 1, this specifies the number of documents the term must appear in;
- * if this is a double in [0,1), then this specifies the fraction of documents.
+ * If this is an integer greater than or equal to 1, this specifies the number of documents
+ * the term must appear in; if this is a double in [0,1), then this specifies the fraction of
+ * documents.
*
* Default: 1.0
* @group param
@@ -78,8 +79,8 @@ private[feature] trait CountVectorizerParams extends Params with HasInputCol wit
/**
* Filter to ignore rare words in a document. For each document, terms with
* frequency/count less than the given threshold are ignored.
- * If this is an integer >= 1, then this specifies a count (of times the term must appear
- * in the document);
+ * If this is an integer greater than or equal to 1, then this specifies a count (of times the
+ * term must appear in the document);
* if this is a double in [0,1), then this specifies a fraction (out of the document's token
* count).
*
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
index a8792a35ff4ae2d7a5167a6972bb8ed0cead1611..db432b6fefafff3f56ba70792ca6996b069b3652 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
@@ -52,7 +52,7 @@ class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
def setOutputCol(value: String): this.type = set(outputCol, value)
/**
- * Number of features. Should be > 0.
+ * Number of features. Should be greater than 0.
* (default = 2^18^)
* @group param
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
index 4463aea0097e2b47d40b376956513394ca0242f0..c8760f9dc178f2d60f32f74cc7fd38f5ed750853 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
@@ -41,7 +41,7 @@ class NGram @Since("1.5.0") (@Since("1.5.0") override val uid: String)
def this() = this(Identifiable.randomUID("ngram"))
/**
- * Minimum n-gram length, >= 1.
+ * Minimum n-gram length, greater than or equal to 1.
* Default: 2, bigram features
* @group param
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
index eb0690058013f2795b8ab24d2c8c39bf67229cd1..6e96545c8cb7a92b5cffd45987338b33728b9ef3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
@@ -37,7 +37,7 @@ class Normalizer @Since("1.4.0") (@Since("1.4.0") override val uid: String)
def this() = this(Identifiable.randomUID("normalizer"))
/**
- * Normalization in L^p^ space. Must be >= 1.
+ * Normalization in L^p^ space. Must be greater than equal to 1.
* (default: p = 2)
* @group param
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
index ea401216aec7b37794ae0a604e80e7cd940f695e..ba1380bdda45175bde7995a123dfbe9d14f81c43 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
@@ -33,14 +33,14 @@ import org.apache.spark.sql.types.{DoubleType, NumericType, StructType}
* at most a single one-value per row that indicates the input category index.
* For example with 5 categories, an input value of 2.0 would map to an output vector of
* `[0.0, 0.0, 1.0, 0.0]`.
- * The last category is not included by default (configurable via [[OneHotEncoder!.dropLast]]
+ * The last category is not included by default (configurable via `OneHotEncoder!.dropLast`
* because it makes the vector entries sum up to one, and hence linearly dependent.
* So an input value of 4.0 maps to `[0.0, 0.0, 0.0, 0.0]`.
*
* @note This is different from scikit-learn's OneHotEncoder, which keeps all categories.
* The output vectors are sparse.
*
- * @see [[StringIndexer]] for converting categorical values into category indices
+ * @see `StringIndexer` for converting categorical values into category indices
*/
@Since("1.4.0")
class OneHotEncoder @Since("1.4.0") (@Since("1.4.0") override val uid: String) extends Transformer
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/PCA.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/PCA.scala
index 6e08bf059124cd72a01ba6bdb129782370b1dae9..4143d864d79308eb7c0d9638dea85a1ccd42fb0c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/PCA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/PCA.scala
@@ -63,7 +63,7 @@ private[feature] trait PCAParams extends Params with HasInputCol with HasOutputC
}
/**
- * PCA trains a model to project vectors to a lower dimensional space of the top [[PCA!.k]]
+ * PCA trains a model to project vectors to a lower dimensional space of the top `PCA!.k`
* principal components.
*/
@Since("1.5.0")
@@ -144,7 +144,7 @@ class PCAModel private[ml] (
* Transform a vector by computed Principal Components.
*
* @note Vectors to be transformed must be the same length as the source vectors given
- * to [[PCA.fit()]].
+ * to `PCA.fit()`.
*/
@Since("2.0.0")
override def transform(dataset: Dataset[_]): DataFrame = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
index 4be17da3e9f76fccd651315bd3a16ca0c982e1d2..292f9496a456c439d522ad31e47224e315541e4a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala
@@ -45,7 +45,8 @@ class PolynomialExpansion @Since("1.4.0") (@Since("1.4.0") override val uid: Str
def this() = this(Identifiable.randomUID("poly"))
/**
- * The polynomial degree to expand, which should be >= 1. A value of 1 means no expansion.
+ * The polynomial degree to expand, which should be greater than equal to 1. A value of 1 means
+ * no expansion.
* Default: 2
* @group param
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
index b9e01dde70d8593c86a5840bb2610c0f529819d2..d8f33cd768dcdf415fb6afaf2a6dcdef7fb3d8bd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
@@ -35,7 +35,7 @@ private[feature] trait QuantileDiscretizerBase extends Params
/**
* Number of buckets (quantiles, or categories) into which data points are grouped. Must
- * be >= 2.
+ * be greater than or equal to 2.
*
* See also [[handleInvalid]], which can optionally create an additional bucket for NaN values.
*
@@ -52,7 +52,7 @@ private[feature] trait QuantileDiscretizerBase extends Params
/**
* Relative error (see documentation for
- * [[org.apache.spark.sql.DataFrameStatFunctions.approxQuantile approxQuantile]] for description)
+ * `org.apache.spark.sql.DataFrameStatFunctions.approxQuantile` for description)
* Must be in the range [0, 1].
* default: 0.001
* @group param
@@ -99,7 +99,7 @@ private[feature] trait QuantileDiscretizerBase extends Params
* but NaNs will be counted in a special bucket[4].
*
* Algorithm: The bin ranges are chosen using an approximate algorithm (see the documentation for
- * [[org.apache.spark.sql.DataFrameStatFunctions.approxQuantile approxQuantile]]
+ * `org.apache.spark.sql.DataFrameStatFunctions.approxQuantile`
* for a detailed description). The precision of the approximation can be controlled with the
* `relativeError` parameter. The lower and upper bin bounds will be `-Infinity` and `+Infinity`,
* covering all real values.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/SQLTransformer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/SQLTransformer.scala
index b25fff973c44168add233b5013cb54075171a9f7..65db06c0d608501f7fecb7647fad51a62a559b1a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/SQLTransformer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/SQLTransformer.scala
@@ -32,9 +32,11 @@ import org.apache.spark.sql.types.StructType
* the output, it can be any select clause that Spark SQL supports. Users can also
* use Spark SQL built-in function and UDFs to operate on these selected columns.
* For example, [[SQLTransformer]] supports statements like:
- * - SELECT a, a + b AS a_b FROM __THIS__
- * - SELECT a, SQRT(b) AS b_sqrt FROM __THIS__ where a > 5
- * - SELECT a, b, SUM(c) AS c_sum FROM __THIS__ GROUP BY a, b
+ * {{{
+ * SELECT a, a + b AS a_b FROM __THIS__
+ * SELECT a, SQRT(b) AS b_sqrt FROM __THIS__ where a > 5
+ * SELECT a, b, SUM(c) AS c_sum FROM __THIS__ GROUP BY a, b
+ * }}}
*/
@Since("1.6.0")
class SQLTransformer @Since("1.6.0") (@Since("1.6.0") override val uid: String) extends Transformer
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
index a55816249c74bfc373581b7c34defc637164f7e0..3fcd84c029e61a1c0a1cd9a37a608efd40a83996 100755
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
@@ -52,7 +52,7 @@ class StopWordsRemover @Since("1.5.0") (@Since("1.5.0") override val uid: String
/**
* The words to be filtered out.
* Default: English stop words
- * @see [[StopWordsRemover.loadDefaultStopWords()]]
+ * @see `StopWordsRemover.loadDefaultStopWords()`
* @group param
*/
@Since("1.5.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index 8b155f00017cfbe3f8d86a1b72f05885c0b95230..0a4d31d1654e7fee455d104edc205fcd6264e78d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -60,7 +60,7 @@ private[feature] trait StringIndexerBase extends Params with HasInputCol with Ha
* The indices are in [0, numLabels), ordered by label frequencies.
* So the most frequent label gets index 0.
*
- * @see [[IndexToString]] for the inverse transformation
+ * @see `IndexToString` for the inverse transformation
*/
@Since("1.4.0")
class StringIndexer @Since("1.4.0") (
@@ -116,7 +116,7 @@ object StringIndexer extends DefaultParamsReadable[StringIndexer] {
* @param labels Ordered list of labels, corresponding to indices to be assigned.
*
* @note During transformation, if the input column does not exist,
- * [[StringIndexerModel.transform]] would return the input dataset unmodified.
+ * `StringIndexerModel.transform` would return the input dataset unmodified.
* This is a temporary fix for the case when target labels do not exist during prediction.
*/
@Since("1.4.0")
@@ -247,12 +247,12 @@ object StringIndexerModel extends MLReadable[StringIndexerModel] {
}
/**
- * A [[Transformer]] that maps a column of indices back to a new column of corresponding
+ * A `Transformer` that maps a column of indices back to a new column of corresponding
* string values.
* The index-string mapping is either from the ML attributes of the input column,
* or from user-supplied labels (which take precedence over ML attributes).
*
- * @see [[StringIndexer]] for converting strings into indices
+ * @see `StringIndexer` for converting strings into indices
*/
@Since("1.5.0")
class IndexToString private[ml] (@Since("1.5.0") override val uid: String)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
index 45d8fa94a8f8f60d64d1bd682faf5d3ec4361eb7..cfaf6c0e610b35320228b82a28a52bfb8dfdbb7c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala
@@ -70,7 +70,7 @@ class RegexTokenizer @Since("1.4.0") (@Since("1.4.0") override val uid: String)
def this() = this(Identifiable.randomUID("regexTok"))
/**
- * Minimum token length, >= 0.
+ * Minimum token length, greater than or equal to 0.
* Default: 1, to avoid returning empty strings
* @group param
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
index d1a5c2e82581e1de181000ddd1cb7ffa62b9aca1..d371da762c55d66ae8f3e0976342277c451d1823 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
@@ -41,8 +41,8 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
/**
* Threshold for the number of values a categorical feature can take.
- * If a feature is found to have > maxCategories values, then it is declared continuous.
- * Must be >= 2.
+ * If a feature is found to have {@literal >} maxCategories values, then it is declared
+ * continuous. Must be greater than or equal to 2.
*
* (default = 20)
* @group param
@@ -59,7 +59,7 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
}
/**
- * Class for indexing categorical feature columns in a dataset of [[Vector]].
+ * Class for indexing categorical feature columns in a dataset of `Vector`.
*
* This has 2 usage modes:
* - Automatically identify categorical features (default behavior)
@@ -76,7 +76,8 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
* - Warning: This can cause problems if features are continuous since this will collect ALL
* unique values to the driver.
* - E.g.: Feature 0 has unique values {-1.0, 0.0}, and feature 1 values {1.0, 3.0, 5.0}.
- * If maxCategories >= 3, then both features will be declared categorical.
+ * If maxCategories is greater than or equal to 3, then both features will be declared
+ * categorical.
*
* This returns a model which can transform categorical features to use 0-based indices.
*
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
index 966ccb85d0e0ed5e96e77e9048b631a939980821..e3e462d07e10c8e3955a47bd176d0ea6b11a6606 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
@@ -32,8 +32,8 @@ import org.apache.spark.sql.types.StructType
* This class takes a feature vector and outputs a new feature vector with a subarray of the
* original features.
*
- * The subset of features can be specified with either indices ([[setIndices()]])
- * or names ([[setNames()]]). At least one feature must be selected. Duplicate features
+ * The subset of features can be specified with either indices (`setIndices()`)
+ * or names (`setNames()`). At least one feature must be selected. Duplicate features
* are not allowed, so there can be no overlap between selected indices and names.
*
* The output vector will order features with the selected indices first (in the order given),
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/package-info.java b/mllib/src/main/scala/org/apache/spark/ml/feature/package-info.java
index dcff4245d1d26a43560a30d403ad06f94f350c20..ce7f3350568727c2b3d5e80158a4866187981fb2 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/package-info.java
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/package-info.java
@@ -61,12 +61,12 @@
* createStructField("id", IntegerType, false),
* createStructField("text", StringType, false),
* createStructField("rating", DoubleType, false)));
- * JavaRDD<Row> rowRDD = jsc.parallelize(
+ * JavaRDD<Row> rowRDD = jsc.parallelize(
* Arrays.asList(
* RowFactory.create(0, "Hi I heard about Spark", 3.0),
* RowFactory.create(1, "I wish Java could use case classes", 4.0),
* RowFactory.create(2, "Logistic regression models are neat", 4.0)));
- * Dataset<Row> dataset = jsql.createDataFrame(rowRDD, schema);
+ * Dataset<Row> dataset = jsql.createDataFrame(rowRDD, schema);
* // define feature transformers
* RegexTokenizer tok = new RegexTokenizer()
* .setInputCol("text")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
index 5bd8ebe0987a9bd85e24ca2fff2b565a55e5f2bc..9adb0fa618f29bc7decfd844622ad73b501374e8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
@@ -87,7 +87,7 @@ class Param[T](val parent: String, val name: String, val doc: String, val isVali
def ->(value: T): ParamPair[T] = ParamPair(this, value)
// scalastyle:on
- /** Encodes a param value into JSON, which can be decoded by [[jsonDecode()]]. */
+ /** Encodes a param value into JSON, which can be decoded by `jsonDecode()`. */
def jsonEncode(value: T): String = {
value match {
case x: String =>
@@ -140,7 +140,7 @@ private[ml] object Param {
/**
* :: DeveloperApi ::
- * Factory methods for common validation functions for [[Param.isValid]].
+ * Factory methods for common validation functions for `Param.isValid`.
* The numerical methods only support Int, Long, Float, and Double.
*/
@DeveloperApi
@@ -165,32 +165,39 @@ object ParamValidators {
s" of unexpected input type: ${value.getClass}")
}
- /** Check if value > lowerBound */
+ /**
+ * Check if value is greater than lowerBound
+ */
def gt[T](lowerBound: Double): T => Boolean = { (value: T) =>
getDouble(value) > lowerBound
}
- /** Check if value >= lowerBound */
+ /**
+ * Check if value is greater than or equal to lowerBound
+ */
def gtEq[T](lowerBound: Double): T => Boolean = { (value: T) =>
getDouble(value) >= lowerBound
}
- /** Check if value < upperBound */
+ /**
+ * Check if value is less than upperBound
+ */
def lt[T](upperBound: Double): T => Boolean = { (value: T) =>
getDouble(value) < upperBound
}
- /** Check if value <= upperBound */
+ /**
+ * Check if value is less than or equal to upperBound
+ */
def ltEq[T](upperBound: Double): T => Boolean = { (value: T) =>
getDouble(value) <= upperBound
}
/**
* Check for value in range lowerBound to upperBound.
- * @param lowerInclusive If true, check for value >= lowerBound.
- * If false, check for value > lowerBound.
- * @param upperInclusive If true, check for value <= upperBound.
- * If false, check for value < upperBound.
+ *
+ * @param lowerInclusive if true, range includes value = lowerBound
+ * @param upperInclusive if true, range includes value = upperBound
*/
def inRange[T](
lowerBound: Double,
@@ -203,7 +210,7 @@ object ParamValidators {
lowerValid && upperValid
}
- /** Version of [[inRange()]] which uses inclusive be default: [lowerBound, upperBound] */
+ /** Version of `inRange()` which uses inclusive be default: [lowerBound, upperBound] */
def inRange[T](lowerBound: Double, upperBound: Double): T => Boolean = {
inRange[T](lowerBound, upperBound, lowerInclusive = true, upperInclusive = true)
}
@@ -228,7 +235,7 @@ object ParamValidators {
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Double]]] for Java.
+ * Specialized version of `Param[Double]` for Java.
*/
@DeveloperApi
class DoubleParam(parent: String, name: String, doc: String, isValid: Double => Boolean)
@@ -288,7 +295,7 @@ private[param] object DoubleParam {
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Int]]] for Java.
+ * Specialized version of `Param[Int]` for Java.
*/
@DeveloperApi
class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolean)
@@ -317,7 +324,7 @@ class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolea
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Float]]] for Java.
+ * Specialized version of `Param[Float]` for Java.
*/
@DeveloperApi
class FloatParam(parent: String, name: String, doc: String, isValid: Float => Boolean)
@@ -378,7 +385,7 @@ private object FloatParam {
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Long]]] for Java.
+ * Specialized version of `Param[Long]` for Java.
*/
@DeveloperApi
class LongParam(parent: String, name: String, doc: String, isValid: Long => Boolean)
@@ -407,7 +414,7 @@ class LongParam(parent: String, name: String, doc: String, isValid: Long => Bool
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Boolean]]] for Java.
+ * Specialized version of `Param[Boolean]` for Java.
*/
@DeveloperApi
class BooleanParam(parent: String, name: String, doc: String) // No need for isValid
@@ -430,7 +437,7 @@ class BooleanParam(parent: String, name: String, doc: String) // No need for isV
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Array[String]]]] for Java.
+ * Specialized version of `Param[Array[String]]` for Java.
*/
@DeveloperApi
class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array[String] => Boolean)
@@ -455,7 +462,7 @@ class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Array[Double]]]] for Java.
+ * Specialized version of `Param[Array[Double]]` for Java.
*/
@DeveloperApi
class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array[Double] => Boolean)
@@ -485,7 +492,7 @@ class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array
/**
* :: DeveloperApi ::
- * Specialized version of [[Param[Array[Int]]]] for Java.
+ * Specialized version of `Param[Array[Int]]` for Java.
*/
@DeveloperApi
class IntArrayParam(parent: Params, name: String, doc: String, isValid: Array[Int] => Boolean)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 6d2c59a905ec7a10ac58764139e19277a1c7e2b0..b466e2ed356814d48923be1dc04d990a2368c063 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -355,8 +355,8 @@ object ALSModel extends MLReadable[ALSModel] {
*
* Essentially instead of finding the low-rank approximations to the rating matrix `R`,
* this finds the approximations for a preference matrix `P` where the elements of `P` are 1 if
- * r > 0 and 0 if r <= 0. The ratings then act as 'confidence' values related to strength of
- * indicated user
+ * r is greater than 0 and 0 if r is less than or equal to 0. The ratings then act as 'confidence'
+ * values related to strength of indicated user
* preferences rather than explicit ratings given to items.
*/
@Since("1.3.0")
@@ -878,7 +878,7 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
}
/**
- * Builder for [[RatingBlock]]. [[mutable.ArrayBuilder]] is used to avoid boxing/unboxing.
+ * Builder for [[RatingBlock]]. `mutable.ArrayBuilder` is used to avoid boxing/unboxing.
*/
private[recommendation] class RatingBlockBuilder[@specialized(Int, Long) ID: ClassTag]
extends Serializable {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index d6ad1ea6d10964163cb01f49960d45175fd15d94..af68e7b9d580902e6f16de8863976c9e812b7be8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -185,7 +185,7 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
setDefault(tol -> 1E-6)
/**
- * Suggested depth for treeAggregate (>= 2).
+ * Suggested depth for treeAggregate (greater than or equal to 2).
* If the dimensions of features or the number of partitions are large,
* this param could be adjusted to a larger size.
* Default is 2.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
index 894b6a2ca2041578ab343406a5e614bc685990e3..0b0c46144bfbe064fdcc2940b4181d90c43791b6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
@@ -132,7 +132,8 @@ object DecisionTreeRegressor extends DefaultParamsReadable[DecisionTreeRegressor
}
/**
- * [[http://en.wikipedia.org/wiki/Decision_tree_learning Decision tree]] model for regression.
+ * <a href="http://en.wikipedia.org/wiki/Decision_tree_learning">
+ * Decision tree (Wikipedia)</a> model for regression.
* It supports both continuous and categorical features.
* @param rootNode Root of the decision tree
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
index 6d8159aa3bdcf926dff89120ed127595cf00f5f0..6e62c8d03c708c3a39b08b18bb78127473446b30 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
@@ -229,7 +229,7 @@ class GBTRegressionModel private[ml](
* (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
* and follows the implementation from scikit-learn.
*
- * @see [[DecisionTreeRegressionModel.featureImportances]]
+ * @see `DecisionTreeRegressionModel.featureImportances`
*/
@Since("2.0.0")
lazy val featureImportances: Vector = TreeEnsembleModel.featureImportances(trees, numFeatures)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index 1201ecd5e4e61d54e646972891b8692b6a0249e2..e718cda2623a0b598e10413e79c6c40d5fcd1348 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -131,10 +131,10 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam
* It supports "gaussian", "binomial", "poisson" and "gamma" as family.
* Valid link functions for each family is listed below. The first link function of each family
* is the default one.
- * - "gaussian" -> "identity", "log", "inverse"
- * - "binomial" -> "logit", "probit", "cloglog"
- * - "poisson" -> "log", "identity", "sqrt"
- * - "gamma" -> "inverse", "identity", "log"
+ * - "gaussian" : "identity", "log", "inverse"
+ * - "binomial" : "logit", "probit", "cloglog"
+ * - "poisson" : "log", "identity", "sqrt"
+ * - "gamma" : "inverse", "identity", "log"
*/
@Experimental
@Since("2.0.0")
@@ -1066,7 +1066,7 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
import GeneralizedLinearRegression._
/**
- * Whether the underlying [[WeightedLeastSquares]] using the "normal" solver.
+ * Whether the underlying `WeightedLeastSquares` using the "normal" solver.
*/
private[ml] val isNormalSolver: Boolean = {
diagInvAtWA.length != 1 || diagInvAtWA(0) != 0
@@ -1074,10 +1074,10 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
/**
* Standard error of estimated coefficients and intercept.
- * This value is only available when the underlying [[WeightedLeastSquares]]
+ * This value is only available when the underlying `WeightedLeastSquares`
* using the "normal" solver.
*
- * If [[GeneralizedLinearRegression.fitIntercept]] is set to true,
+ * If `GeneralizedLinearRegression.fitIntercept` is set to true,
* then the last element returned corresponds to the intercept.
*/
@Since("2.0.0")
@@ -1092,10 +1092,10 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
/**
* T-statistic of estimated coefficients and intercept.
- * This value is only available when the underlying [[WeightedLeastSquares]]
+ * This value is only available when the underlying `WeightedLeastSquares`
* using the "normal" solver.
*
- * If [[GeneralizedLinearRegression.fitIntercept]] is set to true,
+ * If `GeneralizedLinearRegression.fitIntercept` is set to true,
* then the last element returned corresponds to the intercept.
*/
@Since("2.0.0")
@@ -1115,10 +1115,10 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
/**
* Two-sided p-value of estimated coefficients and intercept.
- * This value is only available when the underlying [[WeightedLeastSquares]]
+ * This value is only available when the underlying `WeightedLeastSquares`
* using the "normal" solver.
*
- * If [[GeneralizedLinearRegression.fitIntercept]] is set to true,
+ * If `GeneralizedLinearRegression.fitIntercept` is set to true,
* then the last element returned corresponds to the intercept.
*/
@Since("2.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
index 4d274f3a5bbf1b212963b253d0c9faa1d9b391ff..c378a99e3c230526d2c48ac24ac12bc4e577266e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
@@ -56,7 +56,7 @@ private[regression] trait IsotonicRegressionBase extends Params with HasFeatures
final def getIsotonic: Boolean = $(isotonic)
/**
- * Param for the index of the feature if [[featuresCol]] is a vector column (default: `0`), no
+ * Param for the index of the feature if `featuresCol` is a vector column (default: `0`), no
* effect otherwise.
* @group param
*/
@@ -194,7 +194,7 @@ object IsotonicRegression extends DefaultParamsReadable[IsotonicRegression] {
* Model fitted by IsotonicRegression.
* Predicts using a piecewise linear function.
*
- * For detailed rules see [[org.apache.spark.mllib.regression.IsotonicRegressionModel.predict()]].
+ * For detailed rules see `org.apache.spark.mllib.regression.IsotonicRegressionModel.predict()`.
*
* @param oldModel A [[org.apache.spark.mllib.regression.IsotonicRegressionModel]]
* model trained by [[org.apache.spark.mllib.regression.IsotonicRegression]].
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index 19ddf36a718c4fd529ddeca286bcc097e887acae..534ef87ec64ee3bd8429283b9aecb473336fad93 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -60,11 +60,11 @@ private[regression] trait LinearRegressionParams extends PredictorParams
* The learning objective is to minimize the squared error, with regularization.
* The specific squared error loss function used is:
*
- * <p><blockquote>
+ * <blockquote>
* $$
* L = 1/2n ||A coefficients - y||^2^
* $$
- * </blockquote></p>
+ * </blockquote>
*
* This supports multiple types of regularization:
* - none (a.k.a. ordinary least squares)
@@ -118,8 +118,9 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
/**
* Set the ElasticNet mixing parameter.
- * For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.
- * For 0 < alpha < 1, the penalty is a combination of L1 and L2.
+ * For alpha = 0, the penalty is an L2 penalty.
+ * For alpha = 1, it is an L1 penalty.
+ * For alpha in (0,1), the penalty is a combination of L1 and L2.
* Default is 0.0 which is an L2 penalty.
*
* @group setParam
@@ -165,7 +166,7 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
* - "l-bfgs" denotes Limited-memory BFGS which is a limited-memory quasi-Newton
* optimization method.
* - "normal" denotes using Normal Equation as an analytical solution to the linear regression
- * problem. This solver is limited to [[LinearRegression.MAX_FEATURES_FOR_NORMAL_SOLVER]].
+ * problem. This solver is limited to `LinearRegression.MAX_FEATURES_FOR_NORMAL_SOLVER`.
* - "auto" (default) means that the solver algorithm is selected automatically.
* The Normal Equations solver will be used when possible, but this will automatically fall
* back to iterative optimization methods when needed.
@@ -181,7 +182,7 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
setDefault(solver -> "auto")
/**
- * Suggested depth for treeAggregate (>= 2).
+ * Suggested depth for treeAggregate (greater than or equal to 2).
* If the dimensions of features or the number of partitions are large,
* this param could be adjusted to a larger size.
* Default is 2.
@@ -338,12 +339,12 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
/*
Note that in Linear Regression, the objective history (loss + regularization) returned
from optimizer is computed in the scaled space given by the following formula.
- <p><blockquote>
+ <blockquote>
$$
L &= 1/2n||\sum_i w_i(x_i - \bar{x_i}) / \hat{x_i} - (y - \bar{y}) / \hat{y}||^2
+ regTerms \\
$$
- </blockquote></p>
+ </blockquote>
*/
val arrayBuilder = mutable.ArrayBuilder.make[Double]
var state: optimizer.State = null
@@ -414,7 +415,7 @@ object LinearRegression extends DefaultParamsReadable[LinearRegression] {
override def load(path: String): LinearRegression = super.load(path)
/**
- * When using [[LinearRegression.solver]] == "normal", the solver must limit the number of
+ * When using `LinearRegression.solver` == "normal", the solver must limit the number of
* features to at most this number. The entire covariance matrix X^T^X will be collected
* to the driver. This limit helps prevent memory overflow errors.
*/
@@ -584,7 +585,7 @@ class LinearRegressionTrainingSummary private[regression] (
*
* This value is only available when using the "l-bfgs" solver.
*
- * @see [[LinearRegression.solver]]
+ * @see `LinearRegression.solver`
*/
@Since("1.5.0")
val totalIterations = objectiveHistory.length
@@ -624,7 +625,7 @@ class LinearRegressionSummary private[regression] (
* Reference: <a href="http://en.wikipedia.org/wiki/Explained_variation">
* Wikipedia explain variation</a>
*
- * @note This ignores instance weights (setting all to 1.0) from [[LinearRegression.weightCol]].
+ * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
* This will change in later Spark versions.
*/
@Since("1.5.0")
@@ -634,7 +635,7 @@ class LinearRegressionSummary private[regression] (
* Returns the mean absolute error, which is a risk function corresponding to the
* expected value of the absolute error loss or l1-norm loss.
*
- * @note This ignores instance weights (setting all to 1.0) from [[LinearRegression.weightCol]].
+ * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
* This will change in later Spark versions.
*/
@Since("1.5.0")
@@ -644,7 +645,7 @@ class LinearRegressionSummary private[regression] (
* Returns the mean squared error, which is a risk function corresponding to the
* expected value of the squared error loss or quadratic loss.
*
- * @note This ignores instance weights (setting all to 1.0) from [[LinearRegression.weightCol]].
+ * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
* This will change in later Spark versions.
*/
@Since("1.5.0")
@@ -654,7 +655,7 @@ class LinearRegressionSummary private[regression] (
* Returns the root mean squared error, which is defined as the square root of
* the mean squared error.
*
- * @note This ignores instance weights (setting all to 1.0) from [[LinearRegression.weightCol]].
+ * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
* This will change in later Spark versions.
*/
@Since("1.5.0")
@@ -665,7 +666,7 @@ class LinearRegressionSummary private[regression] (
* Reference: <a href="http://en.wikipedia.org/wiki/Coefficient_of_determination">
* Wikipedia coefficient of determination</a>
*
- * @note This ignores instance weights (setting all to 1.0) from [[LinearRegression.weightCol]].
+ * @note This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`.
* This will change in later Spark versions.
*/
@Since("1.5.0")
@@ -711,10 +712,10 @@ class LinearRegressionSummary private[regression] (
* Standard error of estimated coefficients and intercept.
* This value is only available when using the "normal" solver.
*
- * If [[LinearRegression.fitIntercept]] is set to true,
+ * If `LinearRegression.fitIntercept` is set to true,
* then the last element returned corresponds to the intercept.
*
- * @see [[LinearRegression.solver]]
+ * @see `LinearRegression.solver`
*/
lazy val coefficientStandardErrors: Array[Double] = {
if (diagInvAtWA.length == 1 && diagInvAtWA(0) == 0) {
@@ -739,10 +740,10 @@ class LinearRegressionSummary private[regression] (
* T-statistic of estimated coefficients and intercept.
* This value is only available when using the "normal" solver.
*
- * If [[LinearRegression.fitIntercept]] is set to true,
+ * If `LinearRegression.fitIntercept` is set to true,
* then the last element returned corresponds to the intercept.
*
- * @see [[LinearRegression.solver]]
+ * @see `LinearRegression.solver`
*/
lazy val tValues: Array[Double] = {
if (diagInvAtWA.length == 1 && diagInvAtWA(0) == 0) {
@@ -762,10 +763,10 @@ class LinearRegressionSummary private[regression] (
* Two-sided p-value of estimated coefficients and intercept.
* This value is only available when using the "normal" solver.
*
- * If [[LinearRegression.fitIntercept]] is set to true,
+ * If `LinearRegression.fitIntercept` is set to true,
* then the last element returned corresponds to the intercept.
*
- * @see [[LinearRegression.solver]]
+ * @see `LinearRegression.solver`
*/
lazy val pValues: Array[Double] = {
if (diagInvAtWA.length == 1 && diagInvAtWA(0) == 0) {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
index 90d89c51c5740715986af8209cef66fa1ab0faf7..62dd729a2994a461c871ee8ff99e8dcbc4188040 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
@@ -200,7 +200,7 @@ class RandomForestRegressionModel private[ml] (
* (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.)
* and follows the implementation from scikit-learn.
*
- * @see [[DecisionTreeRegressionModel.featureImportances]]
+ * @see `DecisionTreeRegressionModel.featureImportances`
*/
@Since("1.5.0")
lazy val featureImportances: Vector = TreeEnsembleModel.featureImportances(trees, numFeatures)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
index 442f52bf0231df34788580c767b8fde79ab57d60..bc3c86a57c85bef16a636c98b2d59e3229400f87 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
@@ -35,7 +35,7 @@ import org.apache.spark.rdd.RDD
* @param numClasses For classification: labels can take values {0, ..., numClasses - 1}.
* For regression: fixed at 0 (no meaning).
* @param maxBins Maximum number of bins, for all features.
- * @param featureArity Map: categorical feature index --> arity.
+ * @param featureArity Map: categorical feature index to arity.
* I.e., the feature takes values in {0, ..., arity - 1}.
* @param numBins Number of bins for each feature.
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala
index f34a8310ddf1c482d19044770da3d3c3d301fb0f..3e19f271839429d2dfe28dc262bc60d30a6ca5ab 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala
@@ -48,7 +48,7 @@ private[spark] object MetadataUtils {
* If a feature does not have metadata, it is assumed to be continuous.
* If a feature is Nominal, then it must have the number of values
* specified.
- * @return Map: feature index --> number of categories.
+ * @return Map: feature index to number of categories.
* The map's set of keys will be the set of categorical feature indices.
*/
def getCategoricalFeatures(featuresSchema: StructField): Map[Int, Int] = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
index bbb9886391697589fe21bd2b21e1070f4f3b0a94..95f480455ee451cb938523c107a28f4bb4780569 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
@@ -76,7 +76,7 @@ private[util] sealed trait BaseReadWrite {
*/
protected final def sqlContext: SQLContext = sparkSession.sqlContext
- /** Returns the underlying [[SparkContext]]. */
+ /** Returns the underlying `SparkContext`. */
protected final def sc: SparkContext = sparkSession.sparkContext
}
@@ -169,7 +169,7 @@ trait MLWritable {
* This only handles simple [[org.apache.spark.ml.param.Param]] types; e.g., it will not handle
* [[org.apache.spark.sql.Dataset]].
*
- * @see [[DefaultParamsReadable]], the counterpart to this trait
+ * @see `DefaultParamsReadable`, the counterpart to this trait
*/
@DeveloperApi
trait DefaultParamsWritable extends MLWritable { self: Params =>
@@ -238,7 +238,7 @@ trait MLReadable[T] {
* [[org.apache.spark.sql.Dataset]].
*
* @tparam T ML instance type
- * @see [[DefaultParamsWritable]], the counterpart to this trait
+ * @see `DefaultParamsWritable`, the counterpart to this trait
*/
@DeveloperApi
trait DefaultParamsReadable[T] extends MLReadable[T] {
@@ -345,7 +345,7 @@ private[ml] object DefaultParamsReader {
/**
* All info from metadata file.
*
- * @param params paramMap, as a [[JValue]]
+ * @param params paramMap, as a `JValue`
* @param metadata All metadata, including the other fields
* @param metadataJson Full metadata file String (for debugging)
*/
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
index aec1526b55c49ed749f118f30bccf369a6aac59b..5fb04ed0ee9a2ca7430a4291212c0397cc7f5f0d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
@@ -124,7 +124,7 @@ object SVMModel extends Loader[SVMModel] {
/**
* Train a Support Vector Machine (SVM) using Stochastic Gradient Descent. By default L2
- * regularization is used, which can be changed via [[SVMWithSGD.optimizer]].
+ * regularization is used, which can be changed via `SVMWithSGD.optimizer`.
*
* @note Labels used in SVM should be {0, 1}.
*/
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
index 31f51417528b3d35433b540f5143d445a874bef4..336f2fc114309b99834ee3cc8c40fb8af9208bdd 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
@@ -43,9 +43,9 @@ import org.apache.spark.storage.StorageLevel
* @param k the desired number of leaf clusters (default: 4). The actual number could be smaller if
* there are no divisible leaf clusters.
* @param maxIterations the max number of k-means iterations to split clusters (default: 20)
- * @param minDivisibleClusterSize the minimum number of points (if >= 1.0) or the minimum
- * proportion of points (if < 1.0) of a divisible cluster
- * (default: 1)
+ * @param minDivisibleClusterSize the minimum number of points (if greater than or equal 1.0) or
+ * the minimum proportion of points (if less than 1.0) of a divisible
+ * cluster (default: 1)
* @param seed a random seed (default: hash value of the class name)
*
* @see <a href="http://glaros.dtc.umn.edu/gkhome/fetch/papers/docclusterKDDTMW00.pdf">
@@ -101,8 +101,8 @@ class BisectingKMeans private (
def getMaxIterations: Int = this.maxIterations
/**
- * Sets the minimum number of points (if >= `1.0`) or the minimum proportion of points
- * (if < `1.0`) of a divisible cluster (default: 1).
+ * Sets the minimum number of points (if greater than or equal to `1.0`) or the minimum proportion
+ * of points (if less than `1.0`) of a divisible cluster (default: 1).
*/
@Since("1.6.0")
def setMinDivisibleClusterSize(minDivisibleClusterSize: Double): this.type = {
@@ -113,8 +113,8 @@ class BisectingKMeans private (
}
/**
- * Gets the minimum number of points (if >= `1.0`) or the minimum proportion of points
- * (if < `1.0`) of a divisible cluster.
+ * Gets the minimum number of points (if greater than or equal to `1.0`) or the minimum proportion
+ * of points (if less than `1.0`) of a divisible cluster.
*/
@Since("1.6.0")
def getMinDivisibleClusterSize: Double = minDivisibleClusterSize
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
index 6873d4277a8db3cbc75dab8bd817de9ac009af62..10bd8468b35cf344b11986deaec3871b52967682 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
@@ -274,7 +274,7 @@ class GaussianMixture private (
private[clustering] object GaussianMixture {
/**
* Heuristic to distribute the computation of the `MultivariateGaussian`s, approximately when
- * d > 25 except for when k is very small.
+ * d is greater than 25 except for when k is very small.
* @param k Number of topics
* @param d Number of features
*/
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
index 16742bd284e69c5f03b238b6299007146a67bae1..4cb9200030293d0d26bb104faec2797a1fcb73b3 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
@@ -120,11 +120,11 @@ class LDA private (
* - EM
* - Currently only supports symmetric distributions, so all values in the vector should be
* the same.
- * - Values should be > 1.0
+ * - Values should be greater than 1.0
* - default = uniformly (50 / k) + 1, where 50/k is common in LDA libraries and +1 follows
* from Asuncion et al. (2009), who recommend a +1 adjustment for EM.
* - Online
- * - Values should be >= 0
+ * - Values should be greater than or equal to 0
* - default = uniformly (1.0 / k), following the implementation from
* <a href="https://github.com/Blei-Lab/onlineldavb">here</a>.
*/
@@ -195,11 +195,11 @@ class LDA private (
*
* Optimizer-specific parameter settings:
* - EM
- * - Value should be > 1.0
+ * - Value should be greater than 1.0
* - default = 0.1 + 1, where 0.1 gives a small amount of smoothing and +1 follows
* Asuncion et al. (2009), who recommend a +1 adjustment for EM.
* - Online
- * - Value should be >= 0
+ * - Value should be greater than or equal to 0
* - default = (1.0 / k), following the implementation from
* <a href="https://github.com/Blei-Lab/onlineldavb">here</a>.
*/
@@ -216,7 +216,7 @@ class LDA private (
def getBeta: Double = getTopicConcentration
/**
- * Alias for [[setTopicConcentration()]]
+ * Alias for `setTopicConcentration()`
*/
@Since("1.3.0")
def setBeta(beta: Double): this.type = setTopicConcentration(beta)
@@ -261,11 +261,11 @@ class LDA private (
def getCheckpointInterval: Int = checkpointInterval
/**
- * Parameter for set checkpoint interval (>= 1) or disable checkpoint (-1). E.g. 10 means that
- * the cache will get checkpointed every 10 iterations. Checkpointing helps with recovery
- * (when nodes fail). It also helps with eliminating temporary shuffle files on disk, which can be
- * important when LDA is run for many iterations. If the checkpoint directory is not set in
- * [[org.apache.spark.SparkContext]], this setting is ignored. (default = 10)
+ * Parameter for set checkpoint interval (greater than or equal to 1) or disable checkpoint (-1).
+ * E.g. 10 means that the cache will get checkpointed every 10 iterations. Checkpointing helps
+ * with recovery (when nodes fail). It also helps with eliminating temporary shuffle files on
+ * disk, which can be important when LDA is run for many iterations. If the checkpoint directory
+ * is not set in [[org.apache.spark.SparkContext]], this setting is ignored. (default = 10)
*
* @see [[org.apache.spark.SparkContext#setCheckpointDir]]
*/
@@ -321,7 +321,7 @@ class LDA private (
* @param documents RDD of documents, which are term (word) count vectors paired with IDs.
* The term count vectors are "bags of words" with a fixed-size vocabulary
* (where the vocabulary size is the length of the vector).
- * Document IDs must be unique and >= 0.
+ * Document IDs must be unique and greater than or equal to 0.
* @return Inferred LDA model
*/
@Since("1.3.0")
@@ -340,7 +340,7 @@ class LDA private (
}
/**
- * Java-friendly version of [[run()]]
+ * Java-friendly version of `run()`
*/
@Since("1.3.0")
def run(documents: JavaPairRDD[java.lang.Long, Vector]): LDAModel = {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
index 017fbc6feb0d7183e9f25270642310b5c528b270..25ffd8561fe37b0a664ba558540e988420ce03b7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
@@ -171,7 +171,7 @@ abstract class LDAModel private[clustering] extends Saveable {
* The term count vectors are "bags of words" with a fixed-size vocabulary
* (where the vocabulary size is the length of the vector).
* This must use the same vocabulary (ordering of term counts) as in training.
- * Document IDs must be unique and >= 0.
+ * Document IDs must be unique and greater than or equal to 0.
* @return Estimated topic distribution for each document.
* The returned RDD may be zipped with the given RDD, where each returned vector
* is a multinomial distribution over topics.
@@ -392,7 +392,7 @@ class LocalLDAModel private[spark] (
* literature). Returns a vector of zeros for an empty document.
*
* Note this means to allow quick query for single document. For batch documents, please refer
- * to [[topicDistributions()]] to avoid overhead.
+ * to `topicDistributions()` to avoid overhead.
*
* @param document document to predict topic mixture distributions for
* @return topic mixture distribution for the document
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
index 9687fc8804e89fe3d8ffaa298f005f9b02d8791a..96b49bcc0aac14e3f3caea8cf9108dbf5932c06b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
@@ -350,9 +350,9 @@ final class OnlineLDAOptimizer extends LDAOptimizer {
* Mini-batch fraction in (0, 1], which sets the fraction of document sampled and used in
* each iteration.
*
- * @note This should be adjusted in synch with [[LDA.setMaxIterations()]]
+ * @note This should be adjusted in synch with `LDA.setMaxIterations()`
* so the entire corpus is used. Specifically, set both so that
- * maxIterations * miniBatchFraction >= 1.
+ * maxIterations * miniBatchFraction is at least 1.
*
* Default: 0.05, i.e., 5% of total documents.
*/
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
index 92cd7f22dc439a141c690c1ea87d2eefeace777e..9b7cd0427f5ede46e10e6be0eda6696cfa717093 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
@@ -78,7 +78,8 @@ class BinaryClassificationMetrics @Since("1.3.0") (
* Returns the receiver operating characteristic (ROC) curve,
* which is an RDD of (false positive rate, true positive rate)
* with (0.0, 0.0) prepended and (1.0, 1.0) appended to it.
- * @see http://en.wikipedia.org/wiki/Receiver_operating_characteristic
+ * @see <a href="http://en.wikipedia.org/wiki/Receiver_operating_characteristic">
+ * Receiver operating characteristic (Wikipedia)</a>
*/
@Since("1.0.0")
def roc(): RDD[(Double, Double)] = {
@@ -98,7 +99,8 @@ class BinaryClassificationMetrics @Since("1.3.0") (
/**
* Returns the precision-recall curve, which is an RDD of (recall, precision),
* NOT (precision, recall), with (0.0, 1.0) prepended to it.
- * @see http://en.wikipedia.org/wiki/Precision_and_recall
+ * @see <a href="http://en.wikipedia.org/wiki/Precision_and_recall">
+ * Precision and recall (Wikipedia)</a>
*/
@Since("1.0.0")
def pr(): RDD[(Double, Double)] = {
@@ -118,7 +120,7 @@ class BinaryClassificationMetrics @Since("1.3.0") (
* Returns the (threshold, F-Measure) curve.
* @param beta the beta factor in F-Measure computation.
* @return an RDD of (threshold, F-Measure) pairs.
- * @see http://en.wikipedia.org/wiki/F1_score
+ * @see <a href="http://en.wikipedia.org/wiki/F1_score">F1 score (Wikipedia)</a>
*/
@Since("1.0.0")
def fMeasureByThreshold(beta: Double): RDD[(Double, Double)] = createCurve(FMeasure(beta))
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
index e29b51c3a19da17c00050c13020e36940ac55184..b98aa0534152b8924414ff3a1c45379080bb462d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
@@ -30,7 +30,7 @@ import org.apache.spark.rdd.RDD
/**
* Evaluator for ranking algorithms.
*
- * Java users should use [[RankingMetrics$.of]] to create a [[RankingMetrics]] instance.
+ * Java users should use `RankingMetrics$.of` to create a [[RankingMetrics]] instance.
*
* @param predictionAndLabels an RDD of (predicted ranking, ground truth set) pairs.
*/
@@ -41,9 +41,9 @@ class RankingMetrics[T: ClassTag](predictionAndLabels: RDD[(Array[T], Array[T])]
/**
* Compute the average precision of all the queries, truncated at ranking position k.
*
- * If for a query, the ranking algorithm returns n (n < k) results, the precision value will be
- * computed as #(relevant items retrieved) / k. This formula also applies when the size of the
- * ground truth set is less than k.
+ * If for a query, the ranking algorithm returns n (n is less than k) results, the precision
+ * value will be computed as #(relevant items retrieved) / k. This formula also applies when
+ * the size of the ground truth set is less than k.
*
* If a query has an empty ground truth set, zero will be used as precision together with
* a log warning.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala
index be3319d60ce2526d0137ecaf38f5441a1b1ebe8d..5a4c6aef50b7bede372ecbd9ab70e82105d0e2d7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala
@@ -62,7 +62,7 @@ private[evaluation] object Recall extends BinaryClassificationMetricComputer {
* F-Measure. Defined as 0 if both precision and recall are 0. EG in the case that all examples
* are false positives.
* @param beta the beta constant in F-Measure
- * @see http://en.wikipedia.org/wiki/F1_score
+ * @see <a href="http://en.wikipedia.org/wiki/F1_score">F1 score (Wikipedia)</a>
*/
private[evaluation] case class FMeasure(beta: Double) extends BinaryClassificationMetricComputer {
private val beta2 = beta * beta
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
index 3c26d2670841bbce65d0baccaa996bba9aadbca5..dca031477d3b760698a6ee2a37edd9c821971972 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
@@ -28,7 +28,7 @@ import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset
import org.apache.spark.rdd.RDD
/**
- * Generates association rules from a [[RDD[FreqItemset[Item]]]. This method only generates
+ * Generates association rules from a `RDD[FreqItemset[Item]]`. This method only generates
* association rules which have a single item as the consequent.
*
*/
@@ -56,7 +56,7 @@ class AssociationRules private[fpm] (
/**
* Computes the association rules with confidence above [[minConfidence]].
* @param freqItemsets frequent itemset model obtained from [[FPGrowth]]
- * @return a [[Set[Rule[Item]]] containing the association rules.
+ * @return a `Set[Rule[Item]]` containing the association rules.
*
*/
@Since("1.5.0")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
index b53386012280d3a1984163e7a4d21f2a7b8e8468..e3cf0d4979ed43d56a4baa4b4218ec58e6312768 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
@@ -44,7 +44,7 @@ import org.apache.spark.storage.StorageLevel
/**
* Model trained by [[FPGrowth]], which holds frequent itemsets.
- * @param freqItemsets frequent itemset, which is an RDD of [[FreqItemset]]
+ * @param freqItemsets frequent itemset, which is an RDD of `FreqItemset`
* @tparam Item item type
*/
@Since("1.3.0")
@@ -69,7 +69,7 @@ class FPGrowthModel[Item: ClassTag] @Since("1.3.0") (
* - human-readable (JSON) model metadata to path/metadata/
* - Parquet formatted data to path/data/
*
- * The model may be loaded using [[FPGrowthModel.load]].
+ * The model may be loaded using `FPGrowthModel.load`.
*
* @param sc Spark context used to save model data.
* @param path Path specifying the directory in which to save this model.
@@ -309,7 +309,7 @@ object FPGrowth {
/**
* Frequent itemset.
- * @param items items in this itemset. Java users should call [[FreqItemset#javaItems]] instead.
+ * @param items items in this itemset. Java users should call `FreqItemset.javaItems` instead.
* @param freq frequency
* @tparam Item item type
*
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
index a5641672218dd54db7180c26f5e1364665a39c4f..327cb974ef96ccf08e23d163c2cedd369002307b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
@@ -211,7 +211,7 @@ class PrefixSpan private (
}
/**
- * A Java-friendly version of [[run()]] that reads sequences from a [[JavaRDD]] and returns
+ * A Java-friendly version of `run()` that reads sequences from a `JavaRDD` and returns
* frequent sequences in a [[PrefixSpanModel]].
* @param data ordered sequences of itemsets stored as Java Iterable of Iterables
* @tparam Item item type
@@ -366,13 +366,13 @@ object PrefixSpan extends Logging {
* Items are represented by positive integers, and items in each itemset must be distinct and
* ordered.
* we use 0 as the delimiter between itemsets.
- * For example, a sequence `<(12)(31)1>` is represented by `[0, 1, 2, 0, 1, 3, 0, 1, 0]`.
- * The postfix of this sequence w.r.t. to prefix `<1>` is `<(_2)(13)1>`.
+ * For example, a sequence `(12)(31)1` is represented by `[0, 1, 2, 0, 1, 3, 0, 1, 0]`.
+ * The postfix of this sequence w.r.t. to prefix `1` is `(_2)(13)1`.
* We may reuse the original items array `[0, 1, 2, 0, 1, 3, 0, 1, 0]` to represent the postfix,
* and mark the start index of the postfix, which is `2` in this example.
* So the active items in this postfix are `[2, 0, 1, 3, 0, 1, 0]`.
* We also remember the start indices of partial projections, the ones that split an itemset.
- * For example, another possible partial projection w.r.t. `<1>` is `<(_3)1>`.
+ * For example, another possible partial projection w.r.t. `1` is `(_3)1`.
* We remember the start indices of partial projections, which is `[2, 5]` in this example.
* This data structure makes it easier to do projections.
*
@@ -583,7 +583,7 @@ class PrefixSpanModel[Item] @Since("1.5.0") (
* - human-readable (JSON) model metadata to path/metadata/
* - Parquet formatted data to path/data/
*
- * The model may be loaded using [[PrefixSpanModel.load]].
+ * The model may be loaded using `PrefixSpanModel.load`.
*
* @param sc Spark context used to save model data.
* @param path Path specifying the directory in which to save this model.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala
index bb94745f078e88bc1c19b14843a84092809b7b92..7695aabf4313d67e436da3edc11b3a03bb998487 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala
@@ -32,7 +32,7 @@ private[mllib] object EigenValueDecomposition {
*
* @param mul a function that multiplies the symmetric matrix with a DenseVector.
* @param n dimension of the square matrix (maximum Int.MaxValue).
- * @param k number of leading eigenvalues required, 0 < k < n.
+ * @param k number of leading eigenvalues required, where k must be positive and less than n.
* @param tol tolerance of the eigs computation.
* @param maxIterations the maximum number of Arnoldi update iterations.
* @return a dense vector of eigenvalues in descending order and a dense matrix of eigenvectors
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
index c94d7890cf557fb7066a3265f38822de5ad9bc3c..63ea9d3264b0f35ee02a61c8a7c13050e0e3bf69 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
@@ -77,7 +77,7 @@ sealed trait Vector extends Serializable {
/**
* Returns a hash code value for the vector. The hash code is based on its size and its first 128
- * nonzero entries, using a hash algorithm similar to [[java.util.Arrays.hashCode]].
+ * nonzero entries, using a hash algorithm similar to `java.util.Arrays.hashCode`.
*/
override def hashCode(): Int = {
// This is a reference implementation. It calls return in foreachActive, which is slow.
@@ -351,7 +351,7 @@ object Vectors {
}
/**
- * Parses a string resulted from [[Vector.toString]] into a [[Vector]].
+ * Parses a string resulted from `Vector.toString` into a [[Vector]].
*/
@Since("1.1.0")
def parse(s: String): Vector = {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
index 67da88e804da2d0c004c2d8cbede9ffef50d379f..8979707666a2fc4314b0bbae7097f4abfea061ab 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
@@ -88,11 +88,11 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
* convergenceTol is a condition which decides iteration termination.
* The end of iteration is decided based on below logic.
*
- * - If the norm of the new solution vector is >1, the diff of solution vectors
+ * - If the norm of the new solution vector is greater than 1, the diff of solution vectors
* is compared to relative tolerance which means normalizing by the norm of
* the new solution vector.
- * - If the norm of the new solution vector is <=1, the diff of solution vectors
- * is compared to absolute tolerance which is not normalizing.
+ * - If the norm of the new solution vector is less than or equal to 1, the diff of solution
+ * vectors is compared to absolute tolerance which is not normalizing.
*
* Must be between 0.0 and 1.0 inclusively.
*/
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index 6232ff30a747ed475f43bb06cc18b6d104115556..900eec18489c133f4c6c568caf0da9dd5200a832 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -49,8 +49,7 @@ class LBFGS(private var gradient: Gradient, private var updater: Updater)
* Set the number of corrections used in the LBFGS update. Default 10.
* Values of numCorrections less than 3 are not recommended; large values
* of numCorrections will result in excessive computing time.
- * 3 < numCorrections < 10 is recommended.
- * Restriction: numCorrections > 0
+ * numCorrections must be positive, and values from 4 to 9 are generally recommended.
*/
def setNumCorrections(corrections: Int): this.type = {
require(corrections > 0,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala
index b7c9fcfbfe60f2e21bedbbac5c99bcde60cbf692..86632ae335957ce0da796bae72566cebc3d7b4e1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/NNLS.scala
@@ -53,8 +53,13 @@ private[spark] object NNLS {
* projected gradient method. That is, find x minimising ||Ax - b||_2 given A^T A and A^T b.
*
* We solve the problem
- * min_x 1/2 x^T ata x^T - x^T atb
- * subject to x >= 0
+ *
+ * <blockquote>
+ * $$
+ * min_x 1/2 x^T ata x^T - x^T atb
+ * $$
+ * </blockquote>
+ * where x is nonnegative.
*
* The method used is similar to one described by Polyak (B. T. Polyak, The conjugate gradient
* method in extremal problems, Zh. Vychisl. Mat. Mat. Fiz. 9(4)(1969), pp. 94-112) for bound-
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
index aa7dd1aaa60fedd1b7c6caf6c6e77e8536ed0013..142f0ec6b902128b93e93277a3bc5aaabf34e6d1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
@@ -95,9 +95,9 @@ class SimpleUpdater extends Updater {
* The corresponding proximal operator for the L1 norm is the soft-thresholding
* function. That is, each weight component is shrunk towards 0 by shrinkageVal.
*
- * If w > shrinkageVal, set weight component to w-shrinkageVal.
- * If w < -shrinkageVal, set weight component to w+shrinkageVal.
- * If -shrinkageVal < w < shrinkageVal, set weight component to 0.
+ * If w is greater than shrinkageVal, set weight component to w-shrinkageVal.
+ * If w is less than -shrinkageVal, set weight component to w+shrinkageVal.
+ * If w is (-shrinkageVal, shrinkageVal), set weight component to 0.
*
* Equivalently, set weight component to signum(w) * max(0.0, abs(w) - shrinkageVal)
*/
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
index 6d60136ddc38f103c47f900f1333d9ab32a59713..85d4d7f37f2c006284e599c01cc97fc14b835007 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
@@ -249,8 +249,8 @@ object RandomRDDs {
* shape and scale.
*
* @param sc SparkContext used to create the RDD.
- * @param shape shape parameter (> 0) for the gamma distribution
- * @param scale scale parameter (> 0) for the gamma distribution
+ * @param shape shape parameter (greater than 0) for the gamma distribution
+ * @param scale scale parameter (greater than 0) for the gamma distribution
* @param size Size of the RDD.
* @param numPartitions Number of partitions in the RDD (default: `sc.defaultParallelism`).
* @param seed Random seed (default: a random long integer).
@@ -766,8 +766,8 @@ object RandomRDDs {
* gamma distribution with the input shape and scale.
*
* @param sc SparkContext used to create the RDD.
- * @param shape shape parameter (> 0) for the gamma distribution.
- * @param scale scale parameter (> 0) for the gamma distribution.
+ * @param shape shape parameter (greater than 0) for the gamma distribution.
+ * @param scale scale parameter (greater than 0) for the gamma distribution.
* @param numRows Number of Vectors in the RDD.
* @param numCols Number of elements in each Vector.
* @param numPartitions Number of partitions in the RDD (default: `sc.defaultParallelism`)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala
index adb5e51947f6dacec845ee81c917947f53450a1a..365b2a06110f69775dfc13dcfa1c37709db4f54c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/SlidingRDD.scala
@@ -42,8 +42,8 @@ class SlidingRDDPartition[T](val idx: Int, val prev: Partition, val tail: Seq[T]
* @param windowSize the window size, must be greater than 1
* @param step step size for windows
*
- * @see [[org.apache.spark.mllib.rdd.RDDFunctions.sliding(Int, Int)*]]
- * @see [[scala.collection.IterableLike.sliding(Int, Int)*]]
+ * @see `org.apache.spark.mllib.rdd.RDDFunctions.sliding(Int, Int)*`
+ * @see `scala.collection.IterableLike.sliding(Int, Int)*`
*/
private[mllib]
class SlidingRDD[T: ClassTag](@transient val parent: RDD[T], val windowSize: Int, val step: Int)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
index a8b5955a7285d3f737874a758b4d4d6b62e31a95..d17f7047c5b2b81418363330a4f2122da4dfe977 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
@@ -31,7 +31,8 @@ import org.apache.spark.rdd.RDD
* distribution of the sample data and the theoretical distribution we can provide a test for the
* the null hypothesis that the sample data comes from that theoretical distribution.
* For more information on KS Test:
- * @see [[https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test]]
+ * @see <a href="https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test">
+ * Kolmogorov-Smirnov test (Wikipedia)</a>
*
* Implementation note: We seek to implement the KS test with a minimal number of distributed
* passes. We sort the RDD, and then perform the following operations on a per-partition basis:
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTest.scala
index 97c032de7a813b883c63223af5d4c55c6b47877a..d680237bf687fa6c54a77889aa562c04102dd7ea 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTest.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTest.scala
@@ -47,7 +47,7 @@ case class BinarySample @Since("1.6.0") (
* of the observation.
*
* To address novelty affects, the `peacePeriod` specifies a set number of initial
- * [[org.apache.spark.rdd.RDD]] batches of the [[DStream]] to be dropped from significance testing.
+ * [[org.apache.spark.rdd.RDD]] batches of the `DStream` to be dropped from significance testing.
*
* The `windowSize` sets the number of batches each significance test is to be performed over. The
* window is sliding with a stride length of 1 batch. Setting windowSize to 0 will perform
@@ -97,7 +97,7 @@ class StreamingTest @Since("1.6.0") () extends Logging with Serializable {
}
/**
- * Register a [[DStream]] of values for significance testing.
+ * Register a `DStream` of values for significance testing.
*
* @param data stream of BinarySample(key,value) pairs where the key denotes group membership
* (true = experiment, false = control) and the value is the numerical metric to
@@ -114,7 +114,7 @@ class StreamingTest @Since("1.6.0") () extends Logging with Serializable {
}
/**
- * Register a [[JavaDStream]] of values for significance testing.
+ * Register a `JavaDStream` of values for significance testing.
*
* @param data stream of BinarySample(isExperiment,value) pairs where the isExperiment denotes
* group (true = experiment, false = control) and the value is the numerical metric
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala
index ff27f28459e265ca1cd3e0f46d8d53c1086ba448..14ac14d6d61f409b4a110088584bc793810bbf85 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala
@@ -73,7 +73,7 @@ private[stat] sealed trait StreamingTestMethod extends Serializable {
* This test does not assume equal variance between the two samples and does not assume equal
* sample size.
*
- * @see http://en.wikipedia.org/wiki/Welch%27s_t_test
+ * @see <a href="http://en.wikipedia.org/wiki/Welch%27s_t_test">Welch's t-test (Wikipedia)</a>
*/
private[stat] object WelchTTest extends StreamingTestMethod with Logging {
@@ -115,7 +115,7 @@ private[stat] object WelchTTest extends StreamingTestMethod with Logging {
* mean. This test assumes equal variance between the two samples and does not assume equal sample
* size. For unequal variances, Welch's t-test should be used instead.
*
- * @see http://en.wikipedia.org/wiki/Student%27s_t-test
+ * @see <a href="http://en.wikipedia.org/wiki/Student%27s_t-test">Student's t-test (Wikipedia)</a>
*/
private[stat] object StudentTTest extends StreamingTestMethod with Logging {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/DecisionTree.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/DecisionTree.scala
index d846c43cf2913e26ae7e40e15c53ac27db82beff..499c80767aea7cddee1c562be8d9b75509dce0f7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/DecisionTree.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/DecisionTree.scala
@@ -162,7 +162,7 @@ object DecisionTree extends Serializable with Logging {
* @param numClasses Number of classes for classification. Default value of 2.
* @param maxBins Maximum number of bins used for splitting features.
* @param quantileCalculationStrategy Algorithm for calculating quantiles.
- * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n -> k)
+ * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n to k)
* indicates that feature n is categorical with k categories
* indexed from 0: {0, 1, ..., k-1}.
* @return DecisionTreeModel that can be used for prediction.
@@ -192,7 +192,7 @@ object DecisionTree extends Serializable with Logging {
* @param input Training dataset: RDD of [[org.apache.spark.mllib.regression.LabeledPoint]].
* Labels should take values {0, 1, ..., numClasses-1}.
* @param numClasses Number of classes for classification.
- * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n -> k)
+ * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n to k)
* indicates that feature n is categorical with k categories
* indexed from 0: {0, 1, ..., k-1}.
* @param impurity Criterion used for information gain calculation.
@@ -238,7 +238,7 @@ object DecisionTree extends Serializable with Logging {
*
* @param input Training dataset: RDD of [[org.apache.spark.mllib.regression.LabeledPoint]].
* Labels are real numbers.
- * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n -> k)
+ * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n to k)
* indicates that feature n is categorical with k categories
* indexed from 0: {0, 1, ..., k-1}.
* @param impurity Criterion used for information gain calculation.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.scala
index cdeef16135015f3935b6ae37674bcebbdff619ea..3e85678906b335173ab54b59ae6a1d56fca4eb51 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.scala
@@ -74,7 +74,7 @@ class GradientBoostedTrees private[spark] (
}
/**
- * Java-friendly API for [[org.apache.spark.mllib.tree.GradientBoostedTrees!#run]].
+ * Java-friendly API for `org.apache.spark.mllib.tree.GradientBoostedTrees.run`.
*/
@Since("1.2.0")
def run(input: JavaRDD[LabeledPoint]): GradientBoostedTreesModel = {
@@ -89,7 +89,7 @@ class GradientBoostedTrees private[spark] (
* This dataset should be different from the training dataset,
* but it should follow the same distribution.
* E.g., these two datasets could be created from an original dataset
- * by using [[org.apache.spark.rdd.RDD.randomSplit()]]
+ * by using `org.apache.spark.rdd.RDD.randomSplit()`
* @return GradientBoostedTreesModel that can be used for prediction.
*/
@Since("1.4.0")
@@ -106,7 +106,7 @@ class GradientBoostedTrees private[spark] (
}
/**
- * Java-friendly API for [[org.apache.spark.mllib.tree.GradientBoostedTrees!#runWithValidation]].
+ * Java-friendly API for `org.apache.spark.mllib.tree.GradientBoostedTrees.runWithValidation`.
*/
@Since("1.4.0")
def runWithValidation(
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/RandomForest.scala
index 428af214060927d2f36f6085da4f66866e8fc9bc..1f6cb086cefa98effb0359b8b361171a6e5ea9e8 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/RandomForest.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/RandomForest.scala
@@ -53,14 +53,15 @@ import org.apache.spark.util.Utils
* the type of random forest (classification or regression), feature type
* (continuous, categorical), depth of the tree, quantile calculation strategy,
* etc.
- * @param numTrees If 1, then no bootstrapping is used. If > 1, then bootstrapping is done.
+ * @param numTrees If 1, then no bootstrapping is used. If greater than 1, then bootstrapping is
+ * done.
* @param featureSubsetStrategy Number of features to consider for splits at each node.
* Supported values: "auto", "all", "sqrt", "log2", "onethird".
* Supported numerical values: "(0.0-1.0]", "[1-n]".
* If "auto" is set, this parameter is set based on numTrees:
* if numTrees == 1, set to "all";
- * if numTrees > 1 (forest) set to "sqrt" for classification and
- * to "onethird" for regression.
+ * if numTrees is greater than 1 (forest) set to "sqrt" for
+ * classification and to "onethird" for regression.
* If a real value "n" in the range (0, 1.0] is set,
* use n * number of features.
* If an integer value "n" in the range (1, num features) is set,
@@ -111,7 +112,7 @@ object RandomForest extends Serializable with Logging {
* Supported values: "auto", "all", "sqrt", "log2", "onethird".
* If "auto" is set, this parameter is set based on numTrees:
* if numTrees == 1, set to "all";
- * if numTrees > 1 (forest) set to "sqrt".
+ * if numTrees is greater than 1 (forest) set to "sqrt".
* @param seed Random seed for bootstrapping and choosing feature subsets.
* @return RandomForestModel that can be used for prediction.
*/
@@ -134,7 +135,7 @@ object RandomForest extends Serializable with Logging {
* @param input Training dataset: RDD of [[org.apache.spark.mllib.regression.LabeledPoint]].
* Labels should take values {0, 1, ..., numClasses-1}.
* @param numClasses Number of classes for classification.
- * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n -> k)
+ * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n to k)
* indicates that feature n is categorical with k categories
* indexed from 0: {0, 1, ..., k-1}.
* @param numTrees Number of trees in the random forest.
@@ -142,7 +143,7 @@ object RandomForest extends Serializable with Logging {
* Supported values: "auto", "all", "sqrt", "log2", "onethird".
* If "auto" is set, this parameter is set based on numTrees:
* if numTrees == 1, set to "all";
- * if numTrees > 1 (forest) set to "sqrt".
+ * if numTrees is greater than 1 (forest) set to "sqrt".
* @param impurity Criterion used for information gain calculation.
* Supported values: "gini" (recommended) or "entropy".
* @param maxDepth Maximum depth of the tree (e.g. depth 0 means 1 leaf node, depth 1 means
@@ -200,7 +201,7 @@ object RandomForest extends Serializable with Logging {
* Supported values: "auto", "all", "sqrt", "log2", "onethird".
* If "auto" is set, this parameter is set based on numTrees:
* if numTrees == 1, set to "all";
- * if numTrees > 1 (forest) set to "onethird".
+ * if numTrees is greater than 1 (forest) set to "onethird".
* @param seed Random seed for bootstrapping and choosing feature subsets.
* @return RandomForestModel that can be used for prediction.
*/
@@ -222,7 +223,7 @@ object RandomForest extends Serializable with Logging {
*
* @param input Training dataset: RDD of [[org.apache.spark.mllib.regression.LabeledPoint]].
* Labels are real numbers.
- * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n -> k)
+ * @param categoricalFeaturesInfo Map storing arity of categorical features. An entry (n to k)
* indicates that feature n is categorical with k categories
* indexed from 0: {0, 1, ..., k-1}.
* @param numTrees Number of trees in the random forest.
@@ -230,7 +231,7 @@ object RandomForest extends Serializable with Logging {
* Supported values: "auto", "all", "sqrt", "log2", "onethird".
* If "auto" is set, this parameter is set based on numTrees:
* if numTrees == 1, set to "all";
- * if numTrees > 1 (forest) set to "onethird".
+ * if numTrees is greater than 1 (forest) set to "onethird".
* @param impurity Criterion used for information gain calculation.
* The only supported value for regression is "variance".
* @param maxDepth Maximum depth of the tree. (e.g., depth 0 means 1 leaf node, depth 1 means
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/BoostingStrategy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/BoostingStrategy.scala
index d8405d13ce904a9cecf387dfcd148639212a32e3..4334b316cc83a75a68e14fbb6ad6bce4de492486 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/BoostingStrategy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/BoostingStrategy.scala
@@ -36,14 +36,14 @@ import org.apache.spark.mllib.tree.loss.{LogLoss, Loss, SquaredError}
* @param validationTol validationTol is a condition which decides iteration termination when
* runWithValidation is used.
* The end of iteration is decided based on below logic:
- * If the current loss on the validation set is > 0.01, the diff
+ * If the current loss on the validation set is greater than 0.01, the diff
* of validation error is compared to relative tolerance which is
* validationTol * (current loss on the validation set).
- * If the current loss on the validation set is <= 0.01, the diff
- * of validation error is compared to absolute tolerance which is
+ * If the current loss on the validation set is less than or equal to 0.01,
+ * the diff of validation error is compared to absolute tolerance which is
* validationTol * 0.01.
* Ignored when
- * [[org.apache.spark.mllib.tree.GradientBoostedTrees.run()]] is used.
+ * `org.apache.spark.mllib.tree.GradientBoostedTrees.run()` is used.
*/
@Since("1.2.0")
case class BoostingStrategy @Since("1.4.0") (
@@ -92,8 +92,8 @@ object BoostingStrategy {
/**
* Returns default configuration for the boosting algorithm
* @param algo Learning goal. Supported:
- * [[org.apache.spark.mllib.tree.configuration.Algo.Classification]],
- * [[org.apache.spark.mllib.tree.configuration.Algo.Regression]]
+ * `org.apache.spark.mllib.tree.configuration.Algo.Classification`,
+ * `org.apache.spark.mllib.tree.configuration.Algo.Regression`
* @return Configuration for boosting algorithm
*/
@Since("1.3.0")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
index b34e1b1b56c43b540e7d6e75d83405ffea23ef24..58e8f5be7b9f01829dbdd382cc1e2b623f7accff 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
@@ -28,8 +28,8 @@ import org.apache.spark.mllib.tree.impurity.{Entropy, Gini, Impurity, Variance}
/**
* Stores all the configuration options for tree construction
* @param algo Learning goal. Supported:
- * [[org.apache.spark.mllib.tree.configuration.Algo.Classification]],
- * [[org.apache.spark.mllib.tree.configuration.Algo.Regression]]
+ * `org.apache.spark.mllib.tree.configuration.Algo.Classification`,
+ * `org.apache.spark.mllib.tree.configuration.Algo.Regression`
* @param impurity Criterion used for information gain calculation.
* Supported for Classification: [[org.apache.spark.mllib.tree.impurity.Gini]],
* [[org.apache.spark.mllib.tree.impurity.Entropy]].
@@ -43,9 +43,9 @@ import org.apache.spark.mllib.tree.impurity.{Entropy, Gini, Impurity, Variance}
* for choosing how to split on features at each node.
* More bins give higher granularity.
* @param quantileCalculationStrategy Algorithm for calculating quantiles. Supported:
- * [[org.apache.spark.mllib.tree.configuration.QuantileStrategy.Sort]]
+ * `org.apache.spark.mllib.tree.configuration.QuantileStrategy.Sort`
* @param categoricalFeaturesInfo A map storing information about the categorical variables and the
- * number of discrete values they take. An entry (n -> k)
+ * number of discrete values they take. An entry (n to k)
* indicates that feature n is categorical with k categories
* indexed from 0: {0, 1, ..., k-1}.
* @param minInstancesPerNode Minimum number of instances each child must have after split.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
index be2704df3444f38b75113b485a41907e8a56bdbb..bda5e662779c79a0ac25d50b0f16f818b994cffd 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
@@ -25,7 +25,7 @@ import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType
* Split applied to a feature
* @param feature feature index
* @param threshold Threshold for continuous feature.
- * Split left if feature <= threshold, else right.
+ * Split left if feature is less than or equal to threshold, else right.
* @param featureType type of feature -- categorical or continuous
* @param categories Split left if categorical feature value is in this set, else right.
*/
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
index 153f9f57faf42b741f5041a5cf34e38c82ebe4a4..594c41c2c74467518bb2ce95f3319309364dd242 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/InternalOutputModes.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
import org.apache.spark.sql.streaming.OutputMode
/**
- * Internal helper class to generate objects representing various [[OutputMode]]s,
+ * Internal helper class to generate objects representing various `OutputMode`s,
*/
private[sql] object InternalOutputModes {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index a821d2ca345793626791332aacc414b6ae0b690d..c362104b26ffbcb49f6e7fd3a8b728664b33eb81 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -74,7 +74,7 @@ object Row {
* It is invalid to use the native primitive interface to retrieve a value that is null, instead a
* user must check `isNullAt` before attempting to retrieve a value that might be null.
*
- * To create a new Row, use [[RowFactory.create()]] in Java or [[Row.apply()]] in Scala.
+ * To create a new Row, use `RowFactory.create()` in Java or `Row.apply()` in Scala.
*
* A [[Row]] object can be constructed by providing field values. Example:
* {{{
@@ -343,7 +343,7 @@ trait Row extends Serializable {
}
/**
- * Returns a Map(name -> value) for the requested fieldNames
+ * Returns a Map consisting of names and values for the requested fieldNames
* For primitive types if value is null it returns 'zero value' specific for primitive
* ie. 0 for Int - use isNullAt to ensure that value is not null
*
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index cecad3b7b4c0a104a7e204e1efeadfd48124e277..4dc06fc9cf09b30021f1d6c5aee7638408516e20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -92,7 +92,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
}
/**
- * The default size of a value of the DecimalType is 8 bytes (precision <= 18) or 16 bytes.
+ * The default size of a value of the DecimalType is 8 bytes when precision is at most 18,
+ * and 16 bytes otherwise.
*/
override def defaultSize: Int = if (precision <= Decimal.MAX_LONG_DIGITS) 8 else 16
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index fa3b2b9de5d5d6d1e13d060d66f5be7d6dcf3df5..e99d7865bda910faff8ef4f6d1da1a60e2c30ad3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -97,7 +97,7 @@ class TypedColumn[-T, U](
}
/**
- * A column that will be computed based on the data in a [[DataFrame]].
+ * A column that will be computed based on the data in a `DataFrame`.
*
* A new column is constructed based on the input columns present in a dataframe:
*
@@ -801,7 +801,7 @@ class Column(val expr: Expression) extends Logging {
/**
* An expression that gets an item at position `ordinal` out of an array,
- * or gets a value by key `key` in a [[MapType]].
+ * or gets a value by key `key` in a `MapType`.
*
* @group expr_ops
* @since 1.3.0
@@ -809,7 +809,7 @@ class Column(val expr: Expression) extends Logging {
def getItem(key: Any): Column = withExpr { UnresolvedExtractValue(expr, Literal(key)) }
/**
- * An expression that gets a field by name in a [[StructType]].
+ * An expression that gets a field by name in a `StructType`.
*
* @group expr_ops
* @since 1.3.0
@@ -1195,92 +1195,92 @@ class Column(val expr: Expression) extends Logging {
class ColumnName(name: String) extends Column(name) {
/**
- * Creates a new [[StructField]] of type boolean.
+ * Creates a new `StructField` of type boolean.
* @since 1.3.0
*/
def boolean: StructField = StructField(name, BooleanType)
/**
- * Creates a new [[StructField]] of type byte.
+ * Creates a new `StructField` of type byte.
* @since 1.3.0
*/
def byte: StructField = StructField(name, ByteType)
/**
- * Creates a new [[StructField]] of type short.
+ * Creates a new `StructField` of type short.
* @since 1.3.0
*/
def short: StructField = StructField(name, ShortType)
/**
- * Creates a new [[StructField]] of type int.
+ * Creates a new `StructField` of type int.
* @since 1.3.0
*/
def int: StructField = StructField(name, IntegerType)
/**
- * Creates a new [[StructField]] of type long.
+ * Creates a new `StructField` of type long.
* @since 1.3.0
*/
def long: StructField = StructField(name, LongType)
/**
- * Creates a new [[StructField]] of type float.
+ * Creates a new `StructField` of type float.
* @since 1.3.0
*/
def float: StructField = StructField(name, FloatType)
/**
- * Creates a new [[StructField]] of type double.
+ * Creates a new `StructField` of type double.
* @since 1.3.0
*/
def double: StructField = StructField(name, DoubleType)
/**
- * Creates a new [[StructField]] of type string.
+ * Creates a new `StructField` of type string.
* @since 1.3.0
*/
def string: StructField = StructField(name, StringType)
/**
- * Creates a new [[StructField]] of type date.
+ * Creates a new `StructField` of type date.
* @since 1.3.0
*/
def date: StructField = StructField(name, DateType)
/**
- * Creates a new [[StructField]] of type decimal.
+ * Creates a new `StructField` of type decimal.
* @since 1.3.0
*/
def decimal: StructField = StructField(name, DecimalType.USER_DEFAULT)
/**
- * Creates a new [[StructField]] of type decimal.
+ * Creates a new `StructField` of type decimal.
* @since 1.3.0
*/
def decimal(precision: Int, scale: Int): StructField =
StructField(name, DecimalType(precision, scale))
/**
- * Creates a new [[StructField]] of type timestamp.
+ * Creates a new `StructField` of type timestamp.
* @since 1.3.0
*/
def timestamp: StructField = StructField(name, TimestampType)
/**
- * Creates a new [[StructField]] of type binary.
+ * Creates a new `StructField` of type binary.
* @since 1.3.0
*/
def binary: StructField = StructField(name, BinaryType)
/**
- * Creates a new [[StructField]] of type array.
+ * Creates a new `StructField` of type array.
* @since 1.3.0
*/
def array(dataType: DataType): StructField = StructField(name, ArrayType(dataType))
/**
- * Creates a new [[StructField]] of type map.
+ * Creates a new `StructField` of type map.
* @since 1.3.0
*/
def map(keyType: DataType, valueType: DataType): StructField =
@@ -1289,13 +1289,13 @@ class ColumnName(name: String) extends Column(name) {
def map(mapType: MapType): StructField = StructField(name, mapType)
/**
- * Creates a new [[StructField]] of type struct.
+ * Creates a new `StructField` of type struct.
* @since 1.3.0
*/
def struct(fields: StructField*): StructField = struct(StructType(fields))
/**
- * Creates a new [[StructField]] of type struct.
+ * Creates a new `StructField` of type struct.
* @since 1.3.0
*/
def struct(structType: StructType): StructField = StructField(name, structType)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index 0d43f09bc54cd7b3bf26a80656b14ede8b3b5645..184c5a11298d97da97793709b2143001d4cf1cd4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.types._
/**
- * Functionality for working with missing data in [[DataFrame]]s.
+ * Functionality for working with missing data in `DataFrame`s.
*
* @since 1.3.1
*/
@@ -36,14 +36,14 @@ import org.apache.spark.sql.types._
final class DataFrameNaFunctions private[sql](df: DataFrame) {
/**
- * Returns a new [[DataFrame]] that drops rows containing any null or NaN values.
+ * Returns a new `DataFrame` that drops rows containing any null or NaN values.
*
* @since 1.3.1
*/
def drop(): DataFrame = drop("any", df.columns)
/**
- * Returns a new [[DataFrame]] that drops rows containing null or NaN values.
+ * Returns a new `DataFrame` that drops rows containing null or NaN values.
*
* If `how` is "any", then drop rows containing any null or NaN values.
* If `how` is "all", then drop rows only if every column is null or NaN for that row.
@@ -53,7 +53,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def drop(how: String): DataFrame = drop(how, df.columns)
/**
- * Returns a new [[DataFrame]] that drops rows containing any null or NaN values
+ * Returns a new `DataFrame` that drops rows containing any null or NaN values
* in the specified columns.
*
* @since 1.3.1
@@ -61,7 +61,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def drop(cols: Array[String]): DataFrame = drop(cols.toSeq)
/**
- * (Scala-specific) Returns a new [[DataFrame]] that drops rows containing any null or NaN values
+ * (Scala-specific) Returns a new `DataFrame` that drops rows containing any null or NaN values
* in the specified columns.
*
* @since 1.3.1
@@ -69,7 +69,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def drop(cols: Seq[String]): DataFrame = drop(cols.size, cols)
/**
- * Returns a new [[DataFrame]] that drops rows containing null or NaN values
+ * Returns a new `DataFrame` that drops rows containing null or NaN values
* in the specified columns.
*
* If `how` is "any", then drop rows containing any null or NaN values in the specified columns.
@@ -80,7 +80,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def drop(how: String, cols: Array[String]): DataFrame = drop(how, cols.toSeq)
/**
- * (Scala-specific) Returns a new [[DataFrame]] that drops rows containing null or NaN values
+ * (Scala-specific) Returns a new `DataFrame` that drops rows containing null or NaN values
* in the specified columns.
*
* If `how` is "any", then drop rows containing any null or NaN values in the specified columns.
@@ -97,7 +97,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
}
/**
- * Returns a new [[DataFrame]] that drops rows containing
+ * Returns a new `DataFrame` that drops rows containing
* less than `minNonNulls` non-null and non-NaN values.
*
* @since 1.3.1
@@ -105,7 +105,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def drop(minNonNulls: Int): DataFrame = drop(minNonNulls, df.columns)
/**
- * Returns a new [[DataFrame]] that drops rows containing
+ * Returns a new `DataFrame` that drops rows containing
* less than `minNonNulls` non-null and non-NaN values in the specified columns.
*
* @since 1.3.1
@@ -113,7 +113,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def drop(minNonNulls: Int, cols: Array[String]): DataFrame = drop(minNonNulls, cols.toSeq)
/**
- * (Scala-specific) Returns a new [[DataFrame]] that drops rows containing less than
+ * (Scala-specific) Returns a new `DataFrame` that drops rows containing less than
* `minNonNulls` non-null and non-NaN values in the specified columns.
*
* @since 1.3.1
@@ -126,21 +126,21 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
}
/**
- * Returns a new [[DataFrame]] that replaces null or NaN values in numeric columns with `value`.
+ * Returns a new `DataFrame` that replaces null or NaN values in numeric columns with `value`.
*
* @since 1.3.1
*/
def fill(value: Double): DataFrame = fill(value, df.columns)
/**
- * Returns a new [[DataFrame]] that replaces null values in string columns with `value`.
+ * Returns a new `DataFrame` that replaces null values in string columns with `value`.
*
* @since 1.3.1
*/
def fill(value: String): DataFrame = fill(value, df.columns)
/**
- * Returns a new [[DataFrame]] that replaces null or NaN values in specified numeric columns.
+ * Returns a new `DataFrame` that replaces null or NaN values in specified numeric columns.
* If a specified column is not a numeric column, it is ignored.
*
* @since 1.3.1
@@ -148,7 +148,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def fill(value: Double, cols: Array[String]): DataFrame = fill(value, cols.toSeq)
/**
- * (Scala-specific) Returns a new [[DataFrame]] that replaces null or NaN values in specified
+ * (Scala-specific) Returns a new `DataFrame` that replaces null or NaN values in specified
* numeric columns. If a specified column is not a numeric column, it is ignored.
*
* @since 1.3.1
@@ -167,7 +167,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
}
/**
- * Returns a new [[DataFrame]] that replaces null values in specified string columns.
+ * Returns a new `DataFrame` that replaces null values in specified string columns.
* If a specified column is not a string column, it is ignored.
*
* @since 1.3.1
@@ -175,7 +175,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def fill(value: String, cols: Array[String]): DataFrame = fill(value, cols.toSeq)
/**
- * (Scala-specific) Returns a new [[DataFrame]] that replaces null values in
+ * (Scala-specific) Returns a new `DataFrame` that replaces null values in
* specified string columns. If a specified column is not a string column, it is ignored.
*
* @since 1.3.1
@@ -194,7 +194,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
}
/**
- * Returns a new [[DataFrame]] that replaces null values.
+ * Returns a new `DataFrame` that replaces null values.
*
* The key of the map is the column name, and the value of the map is the replacement value.
* The value must be of the following type:
@@ -213,7 +213,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
def fill(valueMap: java.util.Map[String, Any]): DataFrame = fill0(valueMap.asScala.toSeq)
/**
- * (Scala-specific) Returns a new [[DataFrame]] that replaces null values.
+ * (Scala-specific) Returns a new `DataFrame` that replaces null values.
*
* The key of the map is the column name, and the value of the map is the replacement value.
* The value must be of the following type: `Int`, `Long`, `Float`, `Double`, `String`, `Boolean`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 5be9a99369997ed867fa8a6af0c133f66cd5431c..1af2f9afea5eb421498a992925e51d8fb8d22b7c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.types.StructType
/**
* Interface used to load a [[Dataset]] from external storage systems (e.g. file systems,
- * key-value stores, etc). Use [[SparkSession.read]] to access this.
+ * key-value stores, etc). Use `SparkSession.read` to access this.
*
* @since 1.4.0
*/
@@ -116,7 +116,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads input in as a [[DataFrame]], for data sources that don't require a path (e.g. external
+ * Loads input in as a `DataFrame`, for data sources that don't require a path (e.g. external
* key-value stores).
*
* @since 1.4.0
@@ -126,7 +126,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads input in as a [[DataFrame]], for data sources that require a path (e.g. data backed by
+ * Loads input in as a `DataFrame`, for data sources that require a path (e.g. data backed by
* a local or distributed file system).
*
* @since 1.4.0
@@ -136,7 +136,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads input in as a [[DataFrame]], for data sources that support multiple paths.
+ * Loads input in as a `DataFrame`, for data sources that support multiple paths.
* Only works if the source is a HadoopFsRelationProvider.
*
* @since 1.6.0
@@ -153,7 +153,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Construct a [[DataFrame]] representing the database table accessible via JDBC URL
+ * Construct a `DataFrame` representing the database table accessible via JDBC URL
* url named table and connection properties.
*
* @since 1.4.0
@@ -163,7 +163,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Construct a [[DataFrame]] representing the database table accessible via JDBC URL
+ * Construct a `DataFrame` representing the database table accessible via JDBC URL
* url named table. Partitions of the table will be retrieved in parallel based on the parameters
* passed to this function.
*
@@ -198,10 +198,10 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Construct a [[DataFrame]] representing the database table accessible via JDBC URL
+ * Construct a `DataFrame` representing the database table accessible via JDBC URL
* url named table using connection properties. The `predicates` parameter gives a list
* expressions suitable for inclusion in WHERE clauses; each one defines one partition
- * of the [[DataFrame]].
+ * of the `DataFrame`.
*
* Don't create too many partitions in parallel on a large cluster; otherwise Spark might crash
* your external database systems.
@@ -240,7 +240,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
/**
* Loads a JSON file (<a href="http://jsonlines.org/">JSON Lines text format or
- * newline-delimited JSON</a>) and returns the result as a [[DataFrame]].
+ * newline-delimited JSON</a>) and returns the result as a `DataFrame`.
* See the documentation on the overloaded `json()` method with varargs for more details.
*
* @since 1.4.0
@@ -252,7 +252,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
/**
* Loads a JSON file (<a href="http://jsonlines.org/">JSON Lines text format or
- * newline-delimited JSON</a>) and returns the result as a [[DataFrame]].
+ * newline-delimited JSON</a>) and returns the result as a `DataFrame`.
*
* This function goes through the input once to determine the input schema. If you know the
* schema in advance, use the version that specifies the schema to avoid the extra scan.
@@ -299,7 +299,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
/**
* Loads a `JavaRDD[String]` storing JSON objects (<a href="http://jsonlines.org/">JSON
* Lines text format or newline-delimited JSON</a>) and returns the result as
- * a [[DataFrame]].
+ * a `DataFrame`.
*
* Unless the schema is specified using [[schema]] function, this function goes through the
* input once to determine the input schema.
@@ -311,7 +311,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
/**
* Loads an `RDD[String]` storing JSON objects (<a href="http://jsonlines.org/">JSON Lines
- * text format or newline-delimited JSON</a>) and returns the result as a [[DataFrame]].
+ * text format or newline-delimited JSON</a>) and returns the result as a `DataFrame`.
*
* Unless the schema is specified using [[schema]] function, this function goes through the
* input once to determine the input schema.
@@ -341,7 +341,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads a CSV file and returns the result as a [[DataFrame]]. See the documentation on the
+ * Loads a CSV file and returns the result as a `DataFrame`. See the documentation on the
* other overloaded `csv()` method for more details.
*
* @since 2.0.0
@@ -352,7 +352,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads a CSV file and returns the result as a [[DataFrame]].
+ * Loads a CSV file and returns the result as a `DataFrame`.
*
* This function will go through the input once to determine the input schema if `inferSchema`
* is enabled. To avoid going through the entire data once, disable `inferSchema` option or
@@ -392,7 +392,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
* indicates a timestamp format. Custom date formats follow the formats at
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
- * `java.sql.Timestamp.valueOf()` and `java.sql.Date.valueOf()` or ISO 8601 format.</li>
* <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
* a record can have.</li>
* <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
@@ -415,7 +414,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
def csv(paths: String*): DataFrame = format("csv").load(paths : _*)
/**
- * Loads a Parquet file, returning the result as a [[DataFrame]]. See the documentation
+ * Loads a Parquet file, returning the result as a `DataFrame`. See the documentation
* on the other overloaded `parquet()` method for more details.
*
* @since 2.0.0
@@ -426,7 +425,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads a Parquet file, returning the result as a [[DataFrame]].
+ * Loads a Parquet file, returning the result as a `DataFrame`.
*
* You can set the following Parquet-specific option(s) for reading Parquet files:
* <ul>
@@ -442,7 +441,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads an ORC file and returns the result as a [[DataFrame]].
+ * Loads an ORC file and returns the result as a `DataFrame`.
*
* @param path input path
* @since 1.5.0
@@ -454,7 +453,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads an ORC file and returns the result as a [[DataFrame]].
+ * Loads an ORC file and returns the result as a `DataFrame`.
*
* @param paths input paths
* @since 2.0.0
@@ -464,7 +463,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
def orc(paths: String*): DataFrame = format("orc").load(paths: _*)
/**
- * Returns the specified table as a [[DataFrame]].
+ * Returns the specified table as a `DataFrame`.
*
* @since 1.4.0
*/
@@ -475,7 +474,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads text files and returns a [[DataFrame]] whose schema starts with a string column named
+ * Loads text files and returns a `DataFrame` whose schema starts with a string column named
* "value", and followed by partitioned columns if there are any. See the documentation on
* the other overloaded `text()` method for more details.
*
@@ -487,7 +486,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
}
/**
- * Loads text files and returns a [[DataFrame]] whose schema starts with a string column named
+ * Loads text files and returns a `DataFrame` whose schema starts with a string column named
* "value", and followed by partitioned columns if there are any.
*
* Each line in the text files is a new row in the resulting DataFrame. For example:
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
index a9a861c4635b2c30b396a9fe23e4bd9b316e6923..89c3a74f4f0676d905c62cc8866fdf4712e99ff2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameStatFunctions.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.util.sketch.{BloomFilter, CountMinSketch}
/**
- * Statistic functions for [[DataFrame]]s.
+ * Statistic functions for `DataFrame`s.
*
* @since 1.4.0
*/
@@ -44,7 +44,9 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* of `x` is close to (p * N).
* More precisely,
*
- * floor((p - err) * N) <= rank(x) <= ceil((p + err) * N).
+ * {{{
+ * floor((p - err) * N) <= rank(x) <= ceil((p + err) * N)
+ * }}}
*
* This method implements a variation of the Greenwald-Khanna algorithm (with some speed
* optimizations).
@@ -55,7 +57,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* @param probabilities a list of quantile probabilities
* Each number must belong to [0, 1].
* For example 0 is the minimum, 0.5 is the median, 1 is the maximum.
- * @param relativeError The relative target precision to achieve (>= 0).
+ * @param relativeError The relative target precision to achieve (greater or equal to 0).
* If set to zero, the exact quantiles are computed, which could be very expensive.
* Note that values greater than 1 are accepted but give the same result as 1.
* @return the approximate quantiles at the given probabilities
@@ -189,7 +191,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* The `support` should be greater than 1e-4.
*
* This function is meant for exploratory data analysis, as we make no guarantee about the
- * backward compatibility of the schema of the resulting [[DataFrame]].
+ * backward compatibility of the schema of the resulting `DataFrame`.
*
* @param cols the names of the columns to search frequent items in.
* @param support The minimum frequency for an item to be considered `frequent`. Should be greater
@@ -236,7 +238,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* Uses a `default` support of 1%.
*
* This function is meant for exploratory data analysis, as we make no guarantee about the
- * backward compatibility of the schema of the resulting [[DataFrame]].
+ * backward compatibility of the schema of the resulting `DataFrame`.
*
* @param cols the names of the columns to search frequent items in.
* @return A Local DataFrame with the Array of frequent items for each column.
@@ -254,7 +256,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* and Papadimitriou.
*
* This function is meant for exploratory data analysis, as we make no guarantee about the
- * backward compatibility of the schema of the resulting [[DataFrame]].
+ * backward compatibility of the schema of the resulting `DataFrame`.
*
* @param cols the names of the columns to search frequent items in.
* @return A Local DataFrame with the Array of frequent items for each column.
@@ -299,7 +301,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* Uses a `default` support of 1%.
*
* This function is meant for exploratory data analysis, as we make no guarantee about the
- * backward compatibility of the schema of the resulting [[DataFrame]].
+ * backward compatibility of the schema of the resulting `DataFrame`.
*
* @param cols the names of the columns to search frequent items in.
* @return A Local DataFrame with the Array of frequent items for each column.
@@ -317,7 +319,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* its fraction as zero.
* @param seed random seed
* @tparam T stratum type
- * @return a new [[DataFrame]] that represents the stratified sample
+ * @return a new `DataFrame` that represents the stratified sample
*
* {{{
* val df = spark.createDataFrame(Seq((1, 1), (1, 2), (2, 1), (2, 1), (2, 3), (3, 2),
@@ -354,7 +356,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* its fraction as zero.
* @param seed random seed
* @tparam T stratum type
- * @return a new [[DataFrame]] that represents the stratified sample
+ * @return a new `DataFrame` that represents the stratified sample
*
* @since 1.5.0
*/
@@ -369,7 +371,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* @param depth depth of the sketch
* @param width width of the sketch
* @param seed random seed
- * @return a [[CountMinSketch]] over column `colName`
+ * @return a `CountMinSketch` over column `colName`
* @since 2.0.0
*/
def countMinSketch(colName: String, depth: Int, width: Int, seed: Int): CountMinSketch = {
@@ -383,7 +385,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* @param eps relative error of the sketch
* @param confidence confidence of the sketch
* @param seed random seed
- * @return a [[CountMinSketch]] over column `colName`
+ * @return a `CountMinSketch` over column `colName`
* @since 2.0.0
*/
def countMinSketch(
@@ -398,7 +400,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* @param depth depth of the sketch
* @param width width of the sketch
* @param seed random seed
- * @return a [[CountMinSketch]] over column `colName`
+ * @return a `CountMinSketch` over column `colName`
* @since 2.0.0
*/
def countMinSketch(col: Column, depth: Int, width: Int, seed: Int): CountMinSketch = {
@@ -412,7 +414,7 @@ final class DataFrameStatFunctions private[sql](df: DataFrame) {
* @param eps relative error of the sketch
* @param confidence confidence of the sketch
* @param seed random seed
- * @return a [[CountMinSketch]] over column `colName`
+ * @return a `CountMinSketch` over column `colName`
* @since 2.0.0
*/
def countMinSketch(col: Column, eps: Double, confidence: Double, seed: Int): CountMinSketch = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 8294e4149b1c4809c7ffb7b98ae29c36bce1a81d..fa8e8cb985ef570d55aee807301fa1139a35f473 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.types.StructType
/**
* Interface used to write a [[Dataset]] to external storage systems (e.g. file systems,
- * key-value stores, etc). Use [[Dataset.write]] to access this.
+ * key-value stores, etc). Use `Dataset.write` to access this.
*
* @since 1.4.0
*/
@@ -189,7 +189,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] at the specified path.
+ * Saves the content of the `DataFrame` at the specified path.
*
* @since 1.4.0
*/
@@ -199,7 +199,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] as the specified table.
+ * Saves the content of the `DataFrame` as the specified table.
*
* @since 1.4.0
*/
@@ -215,8 +215,8 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
dataSource.write(mode, df)
}
/**
- * Inserts the content of the [[DataFrame]] to the specified table. It requires that
- * the schema of the [[DataFrame]] is the same as the schema of the table.
+ * Inserts the content of the `DataFrame` to the specified table. It requires that
+ * the schema of the `DataFrame` is the same as the schema of the table.
*
* @note Unlike `saveAsTable`, `insertInto` ignores the column names and just uses position-based
* resolution. For example:
@@ -322,15 +322,15 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] as the specified table.
+ * Saves the content of the `DataFrame` as the specified table.
*
* In the case the table already exists, behavior of this function depends on the
* save mode, specified by the `mode` function (default to throwing an exception).
- * When `mode` is `Overwrite`, the schema of the [[DataFrame]] does not need to be
+ * When `mode` is `Overwrite`, the schema of the `DataFrame` does not need to be
* the same as that of the existing table.
*
* When `mode` is `Append`, if there is an existing table, we will use the format and options of
- * the existing table. The column order in the schema of the [[DataFrame]] doesn't need to be same
+ * the existing table. The column order in the schema of the `DataFrame` doesn't need to be same
* as that of the existing table. Unlike `insertInto`, `saveAsTable` will use the column names to
* find the correct column positions. For example:
*
@@ -346,7 +346,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* +---+---+
* }}}
*
- * When the DataFrame is created from a non-partitioned [[HadoopFsRelation]] with a single input
+ * When the DataFrame is created from a non-partitioned `HadoopFsRelation` with a single input
* path, and the data source provider can be mapped to an existing Hive builtin SerDe (i.e. ORC
* and Parquet), the table is persisted in a Hive compatible format, which means other systems
* like Hive will be able to read this table. Otherwise, the table is persisted in a Spark SQL
@@ -406,7 +406,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] to an external database table via JDBC. In the case the
+ * Saves the content of the `DataFrame` to an external database table via JDBC. In the case the
* table already exists in the external database, behavior of this function depends on the
* save mode, specified by the `mode` function (default to throwing an exception).
*
@@ -447,7 +447,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] in JSON format (<a href="http://jsonlines.org/">
+ * Saves the content of the `DataFrame` in JSON format (<a href="http://jsonlines.org/">
* JSON Lines text format or newline-delimited JSON</a>) at the specified path.
* This is equivalent to:
* {{{
@@ -474,7 +474,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] in Parquet format at the specified path.
+ * Saves the content of the `DataFrame` in Parquet format at the specified path.
* This is equivalent to:
* {{{
* format("parquet").save(path)
@@ -495,7 +495,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] in ORC format at the specified path.
+ * Saves the content of the `DataFrame` in ORC format at the specified path.
* This is equivalent to:
* {{{
* format("orc").save(path)
@@ -516,7 +516,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] in a text file at the specified path.
+ * Saves the content of the `DataFrame` in a text file at the specified path.
* The DataFrame must have only one column that is of string type.
* Each row becomes a new line in the output file. For example:
* {{{
@@ -541,7 +541,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
/**
- * Saves the content of the [[DataFrame]] in CSV format at the specified path.
+ * Saves the content of the `DataFrame` in CSV format at the specified path.
* This is equivalent to:
* {{{
* format("csv").save(path)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 7ba6ffce278cf558968d79f6e87a5914e485ba45..fcc02e5eb3ef9fd4ba11837a991b9b708d6269d2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -68,7 +68,7 @@ private[sql] object Dataset {
/**
* A Dataset is a strongly typed collection of domain-specific objects that can be transformed
* in parallel using functional or relational operations. Each Dataset also has an untyped view
- * called a [[DataFrame]], which is a Dataset of [[Row]].
+ * called a `DataFrame`, which is a Dataset of [[Row]].
*
* Operations available on Datasets are divided into transformations and actions. Transformations
* are the ones that produce new Datasets, and actions are the ones that trigger computation and
@@ -363,7 +363,7 @@ class Dataset[T] private[sql](
* - When `U` is a tuple, the columns will be be mapped by ordinal (i.e. the first column will
* be assigned to `_1`).
* - When `U` is a primitive type (i.e. String, Int, etc), then the first column of the
- * [[DataFrame]] will be used.
+ * `DataFrame` will be used.
*
* If the schema of the Dataset does not match the desired `U` type, you can use `select`
* along with `alias` or `as` to rearrange or rename as required.
@@ -377,7 +377,7 @@ class Dataset[T] private[sql](
/**
* Converts this strongly typed collection of data to generic `DataFrame` with columns renamed.
- * This can be quite convenient in conversion from an RDD of tuples into a [[DataFrame]] with
+ * This can be quite convenient in conversion from an RDD of tuples into a `DataFrame` with
* meaningful names. For example:
* {{{
* val rdd: RDD[(Int, String)] = ...
@@ -472,8 +472,8 @@ class Dataset[T] private[sql](
/**
* Returns true if this Dataset contains one or more sources that continuously
* return data as it arrives. A Dataset that reads data from a streaming source
- * must be executed as a [[StreamingQuery]] using the `start()` method in
- * [[DataStreamWriter]]. Methods that return a single answer, e.g. `count()` or
+ * must be executed as a `StreamingQuery` using the `start()` method in
+ * `DataStreamWriter`. Methods that return a single answer, e.g. `count()` or
* `collect()`, will throw an [[AnalysisException]] when there is a streaming
* source present.
*
@@ -685,7 +685,7 @@ class Dataset[T] private[sql](
def stat: DataFrameStatFunctions = new DataFrameStatFunctions(toDF())
/**
- * Join with another [[DataFrame]].
+ * Join with another `DataFrame`.
*
* Behaves as an INNER JOIN and requires a subsequent join predicate.
*
@@ -699,7 +699,7 @@ class Dataset[T] private[sql](
}
/**
- * Inner equi-join with another [[DataFrame]] using the given column.
+ * Inner equi-join with another `DataFrame` using the given column.
*
* Different from other join functions, the join column will only appear once in the output,
* i.e. similar to SQL's `JOIN USING` syntax.
@@ -713,7 +713,7 @@ class Dataset[T] private[sql](
* @param usingColumn Name of the column to join on. This column must exist on both sides.
*
* @note If you perform a self-join using this function without aliasing the input
- * [[DataFrame]]s, you will NOT be able to reference any columns after the join, since
+ * `DataFrame`s, you will NOT be able to reference any columns after the join, since
* there is no way to disambiguate which side of the join you would like to reference.
*
* @group untypedrel
@@ -724,7 +724,7 @@ class Dataset[T] private[sql](
}
/**
- * Inner equi-join with another [[DataFrame]] using the given columns.
+ * Inner equi-join with another `DataFrame` using the given columns.
*
* Different from other join functions, the join columns will only appear once in the output,
* i.e. similar to SQL's `JOIN USING` syntax.
@@ -738,7 +738,7 @@ class Dataset[T] private[sql](
* @param usingColumns Names of the columns to join on. This columns must exist on both sides.
*
* @note If you perform a self-join using this function without aliasing the input
- * [[DataFrame]]s, you will NOT be able to reference any columns after the join, since
+ * `DataFrame`s, you will NOT be able to reference any columns after the join, since
* there is no way to disambiguate which side of the join you would like to reference.
*
* @group untypedrel
@@ -749,7 +749,7 @@ class Dataset[T] private[sql](
}
/**
- * Equi-join with another [[DataFrame]] using the given columns.
+ * Equi-join with another `DataFrame` using the given columns.
*
* Different from other join functions, the join columns will only appear once in the output,
* i.e. similar to SQL's `JOIN USING` syntax.
@@ -759,7 +759,7 @@ class Dataset[T] private[sql](
* @param joinType One of: `inner`, `outer`, `left_outer`, `right_outer`, `leftsemi`.
*
* @note If you perform a self-join using this function without aliasing the input
- * [[DataFrame]]s, you will NOT be able to reference any columns after the join, since
+ * `DataFrame`s, you will NOT be able to reference any columns after the join, since
* there is no way to disambiguate which side of the join you would like to reference.
*
* @group untypedrel
@@ -782,7 +782,7 @@ class Dataset[T] private[sql](
}
/**
- * Inner join with another [[DataFrame]], using the given join expression.
+ * Inner join with another `DataFrame`, using the given join expression.
*
* {{{
* // The following two are equivalent:
@@ -796,7 +796,7 @@ class Dataset[T] private[sql](
def join(right: Dataset[_], joinExprs: Column): DataFrame = join(right, joinExprs, "inner")
/**
- * Join with another [[DataFrame]], using the given join expression. The following performs
+ * Join with another `DataFrame`, using the given join expression. The following performs
* a full outer join between `df1` and `df2`.
*
* {{{
@@ -860,7 +860,7 @@ class Dataset[T] private[sql](
}
/**
- * Explicit cartesian join with another [[DataFrame]].
+ * Explicit cartesian join with another `DataFrame`.
*
* @param right Right side of the join operation.
*
@@ -875,7 +875,7 @@ class Dataset[T] private[sql](
/**
* :: Experimental ::
- * Joins this Dataset returning a [[Tuple2]] for each pair where `condition` evaluates to
+ * Joins this Dataset returning a `Tuple2` for each pair where `condition` evaluates to
* true.
*
* This is similar to the relation `join` function with one important difference in the
@@ -956,7 +956,7 @@ class Dataset[T] private[sql](
/**
* :: Experimental ::
- * Using inner equi-join to join this Dataset returning a [[Tuple2]] for each pair
+ * Using inner equi-join to join this Dataset returning a `Tuple2` for each pair
* where `condition` evaluates to true.
*
* @param other Right side of the join.
@@ -2232,7 +2232,7 @@ class Dataset[T] private[sql](
}
/**
- * Returns a new [[DataFrame]] that contains the result of applying a serialized R function
+ * Returns a new `DataFrame` that contains the result of applying a serialized R function
* `func` to each partition.
*/
private[sql] def mapPartitionsInR(
@@ -2446,7 +2446,7 @@ class Dataset[T] private[sql](
/**
* Returns a new Dataset that has exactly `numPartitions` partitions.
- * Similar to coalesce defined on an [[RDD]], this operation results in a narrow dependency, e.g.
+ * Similar to coalesce defined on an `RDD`, this operation results in a narrow dependency, e.g.
* if you go from 1000 partitions to 100 partitions, there will not be a shuffle, instead each of
* the 100 new partitions will claim 10 of the current partitions.
*
@@ -2536,7 +2536,7 @@ class Dataset[T] private[sql](
def unpersist(): this.type = unpersist(blocking = false)
/**
- * Represents the content of the Dataset as an [[RDD]] of [[T]].
+ * Represents the content of the Dataset as an `RDD` of [[T]].
*
* @group basic
* @since 1.6.0
@@ -2550,14 +2550,14 @@ class Dataset[T] private[sql](
}
/**
- * Returns the content of the Dataset as a [[JavaRDD]] of [[T]]s.
+ * Returns the content of the Dataset as a `JavaRDD` of [[T]]s.
* @group basic
* @since 1.6.0
*/
def toJavaRDD: JavaRDD[T] = rdd.toJavaRDD()
/**
- * Returns the content of the Dataset as a [[JavaRDD]] of [[T]]s.
+ * Returns the content of the Dataset as a `JavaRDD` of [[T]]s.
* @group basic
* @since 1.6.0
*/
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
index 1163035e315fc04de8a01a7bd11dbe11f4c47cd2..b94ad59fa2f6ef88b232ec783c7849d79bc5de62 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
@@ -18,11 +18,10 @@
package org.apache.spark.sql
import org.apache.spark.annotation.{Experimental, InterfaceStability}
-import org.apache.spark.sql.streaming.StreamingQuery
/**
* :: Experimental ::
- * A class to consume data generated by a [[StreamingQuery]]. Typically this is used to send the
+ * A class to consume data generated by a `StreamingQuery`. Typically this is used to send the
* generated data to external systems. Each partition will use a new deserialized instance, so you
* usually should do all the initialization (e.g. opening a connection or initiating a transaction)
* in the `open` method.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
index 31ce8eb25e80877d15059d554156920fbd323e6e..395d709f265913fbad5c5e4eda6e462567856c2a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
@@ -131,7 +131,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
* This function does not support partial aggregation, and as a result requires shuffling all
* the data in the [[Dataset]]. If an application intends to perform an aggregation over each
* key, it is best to use the reduce function or an
- * [[org.apache.spark.sql.expressions#Aggregator Aggregator]].
+ * `org.apache.spark.sql.expressions#Aggregator`.
*
* Internally, the implementation will spill to disk if any given group is too large to fit into
* memory. However, users must take care to avoid materializing the whole iterator for a group
@@ -160,7 +160,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
* This function does not support partial aggregation, and as a result requires shuffling all
* the data in the [[Dataset]]. If an application intends to perform an aggregation over each
* key, it is best to use the reduce function or an
- * [[org.apache.spark.sql.expressions#Aggregator Aggregator]].
+ * `org.apache.spark.sql.expressions#Aggregator`.
*
* Internally, the implementation will spill to disk if any given group is too large to fit into
* memory. However, users must take care to avoid materializing the whole iterator for a group
@@ -182,7 +182,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
* This function does not support partial aggregation, and as a result requires shuffling all
* the data in the [[Dataset]]. If an application intends to perform an aggregation over each
* key, it is best to use the reduce function or an
- * [[org.apache.spark.sql.expressions#Aggregator Aggregator]].
+ * `org.apache.spark.sql.expressions#Aggregator`.
*
* Internally, the implementation will spill to disk if any given group is too large to fit into
* memory. However, users must take care to avoid materializing the whole iterator for a group
@@ -205,7 +205,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
* This function does not support partial aggregation, and as a result requires shuffling all
* the data in the [[Dataset]]. If an application intends to perform an aggregation over each
* key, it is best to use the reduce function or an
- * [[org.apache.spark.sql.expressions#Aggregator Aggregator]].
+ * `org.apache.spark.sql.expressions#Aggregator`.
*
* Internally, the implementation will spill to disk if any given group is too large to fit into
* memory. However, users must take care to avoid materializing the whole iterator for a group
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index f019d1e9dacebe85887f52206296a914f371c2a2..0fe8d87ebd6ba7226ff58eaa5c0b6b8a6d91d0ff 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.types.NumericType
import org.apache.spark.sql.types.StructType
/**
- * A set of methods for aggregations on a [[DataFrame]], created by [[Dataset.groupBy]].
+ * A set of methods for aggregations on a `DataFrame`, created by `Dataset.groupBy`.
*
* The main method is the agg function, which has multiple variants. This class also contains
* convenience some first order statistics such as mean, sum for convenience.
@@ -129,7 +129,7 @@ class RelationalGroupedDataset protected[sql](
/**
* (Scala-specific) Compute aggregates by specifying the column names and
- * aggregate methods. The resulting [[DataFrame]] will also contain the grouping columns.
+ * aggregate methods. The resulting `DataFrame` will also contain the grouping columns.
*
* The available aggregate methods are `avg`, `max`, `min`, `sum`, `count`.
* {{{
@@ -150,7 +150,7 @@ class RelationalGroupedDataset protected[sql](
/**
* (Scala-specific) Compute aggregates by specifying a map from column name to
- * aggregate methods. The resulting [[DataFrame]] will also contain the grouping columns.
+ * aggregate methods. The resulting `DataFrame` will also contain the grouping columns.
*
* The available aggregate methods are `avg`, `max`, `min`, `sum`, `count`.
* {{{
@@ -171,7 +171,7 @@ class RelationalGroupedDataset protected[sql](
/**
* (Java-specific) Compute aggregates by specifying a map from column name to
- * aggregate methods. The resulting [[DataFrame]] will also contain the grouping columns.
+ * aggregate methods. The resulting `DataFrame` will also contain the grouping columns.
*
* The available aggregate methods are `avg`, `max`, `min`, `sum`, `count`.
* {{{
@@ -228,7 +228,7 @@ class RelationalGroupedDataset protected[sql](
/**
* Count the number of rows for each group.
- * The resulting [[DataFrame]] will also contain the grouping columns.
+ * The resulting `DataFrame` will also contain the grouping columns.
*
* @since 1.3.0
*/
@@ -236,7 +236,7 @@ class RelationalGroupedDataset protected[sql](
/**
* Compute the average value for each numeric columns for each group. This is an alias for `avg`.
- * The resulting [[DataFrame]] will also contain the grouping columns.
+ * The resulting `DataFrame` will also contain the grouping columns.
* When specified columns are given, only compute the average values for them.
*
* @since 1.3.0
@@ -248,7 +248,7 @@ class RelationalGroupedDataset protected[sql](
/**
* Compute the max value for each numeric columns for each group.
- * The resulting [[DataFrame]] will also contain the grouping columns.
+ * The resulting `DataFrame` will also contain the grouping columns.
* When specified columns are given, only compute the max values for them.
*
* @since 1.3.0
@@ -260,7 +260,7 @@ class RelationalGroupedDataset protected[sql](
/**
* Compute the mean value for each numeric columns for each group.
- * The resulting [[DataFrame]] will also contain the grouping columns.
+ * The resulting `DataFrame` will also contain the grouping columns.
* When specified columns are given, only compute the mean values for them.
*
* @since 1.3.0
@@ -272,7 +272,7 @@ class RelationalGroupedDataset protected[sql](
/**
* Compute the min value for each numeric column for each group.
- * The resulting [[DataFrame]] will also contain the grouping columns.
+ * The resulting `DataFrame` will also contain the grouping columns.
* When specified columns are given, only compute the min values for them.
*
* @since 1.3.0
@@ -284,7 +284,7 @@ class RelationalGroupedDataset protected[sql](
/**
* Compute the sum for each numeric columns for each group.
- * The resulting [[DataFrame]] will also contain the grouping columns.
+ * The resulting `DataFrame` will also contain the grouping columns.
* When specified columns are given, only compute the sum for them.
*
* @since 1.3.0
@@ -295,7 +295,7 @@ class RelationalGroupedDataset protected[sql](
}
/**
- * Pivots a column of the current [[DataFrame]] and perform the specified aggregation.
+ * Pivots a column of the current `DataFrame` and perform the specified aggregation.
* There are two versions of pivot function: one that requires the caller to specify the list
* of distinct values to pivot on, and one that does not. The latter is more concise but less
* efficient, because Spark needs to first compute the list of distinct values internally.
@@ -335,7 +335,7 @@ class RelationalGroupedDataset protected[sql](
}
/**
- * Pivots a column of the current [[DataFrame]] and perform the specified aggregation.
+ * Pivots a column of the current `DataFrame` and perform the specified aggregation.
* There are two versions of pivot function: one that requires the caller to specify the list
* of distinct values to pivot on, and one that does not. The latter is more concise but less
* efficient, because Spark needs to first compute the list of distinct values internally.
@@ -367,7 +367,7 @@ class RelationalGroupedDataset protected[sql](
}
/**
- * Pivots a column of the current [[DataFrame]] and perform the specified aggregation.
+ * Pivots a column of the current `DataFrame` and perform the specified aggregation.
* There are two versions of pivot function: one that requires the caller to specify the list
* of distinct values to pivot on, and one that does not. The latter is more concise but less
* efficient, because Spark needs to first compute the list of distinct values internally.
@@ -392,12 +392,12 @@ class RelationalGroupedDataset protected[sql](
* Applies the given serialized R function `func` to each group of data. For each unique group,
* the function will be passed the group key and an iterator that contains all of the elements in
* the group. The function can return an iterator containing elements of an arbitrary type which
- * will be returned as a new [[DataFrame]].
+ * will be returned as a new `DataFrame`.
*
* This function does not support partial aggregation, and as a result requires shuffling all
* the data in the [[Dataset]]. If an application intends to perform an aggregation over each
* key, it is best to use the reduce function or an
- * [[org.apache.spark.sql.expressions#Aggregator Aggregator]].
+ * `org.apache.spark.sql.expressions#Aggregator`.
*
* Internally, the implementation will spill to disk if any given group is too large to fit into
* memory. However, users must take care to avoid materializing the whole iterator for a group
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index 9108d19d0a0c2589e4d1a8f90d3b26f10a395f2a..edfcd7d56dc8be89b2cf4ecc747f93252c2e83c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
/**
- * Runtime configuration interface for Spark. To access this, use [[SparkSession.conf]].
+ * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`.
*
* Options set here are automatically propagated to the Hadoop configuration during I/O.
*
@@ -65,7 +65,8 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
/**
* Returns the value of Spark runtime configuration property for the given key.
*
- * @throws NoSuchElementException if the key is not set and does not have a default value
+ * @throws java.util.NoSuchElementException if the key is not set and does not have a default
+ * value
* @since 2.0.0
*/
@throws[NoSuchElementException]("if the key is not set")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 858fa4c7609b6c29e2bdbed485d48dfd7ea242d8..6554359806a0189a16b78a6588ee5a28e9ef03f8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -84,7 +84,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
/**
* Returns a [[SQLContext]] as new session, with separated SQL configurations, temporary
- * tables, registered functions, but sharing the same [[SparkContext]], cached data and
+ * tables, registered functions, but sharing the same `SparkContext`, cached data and
* other things.
*
* @since 1.6.0
@@ -883,7 +883,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
}
/**
- * Loads an JavaRDD<String> storing JSON objects (one object per record) and applies the given
+ * Loads an JavaRDD[String] storing JSON objects (one object per record) and applies the given
* schema, returning the result as a `DataFrame`.
*
* @group specificdata
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 71b1880dc0715ad4dbcc52edcf5b33562a4fd716..08d74ac0185b8c97f225cb14acb62225f794c4a9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -93,7 +93,7 @@ class SparkSession private(
* ----------------------- */
/**
- * State shared across sessions, including the [[SparkContext]], cached data, listener,
+ * State shared across sessions, including the `SparkContext`, cached data, listener,
* and a catalog that interacts with external systems.
*/
@transient
@@ -125,7 +125,7 @@ class SparkSession private(
*
* This is the interface through which the user can get and set all Spark and Hadoop
* configurations that are relevant to Spark SQL. When getting the value of a config,
- * this defaults to the value set in the underlying [[SparkContext]], if any.
+ * this defaults to the value set in the underlying `SparkContext`, if any.
*
* @since 2.0.0
*/
@@ -189,8 +189,8 @@ class SparkSession private(
/**
* :: Experimental ::
- * Returns a [[StreamingQueryManager]] that allows managing all the
- * [[StreamingQuery StreamingQueries]] active on `this`.
+ * Returns a `StreamingQueryManager` that allows managing all the
+ * `StreamingQuery`s active on `this`.
*
* @since 2.0.0
*/
@@ -200,9 +200,9 @@ class SparkSession private(
/**
* Start a new session with isolated SQL configurations, temporary tables, registered
- * functions are isolated, but sharing the underlying [[SparkContext]] and cached data.
+ * functions are isolated, but sharing the underlying `SparkContext` and cached data.
*
- * @note Other than the [[SparkContext]], all shared state is initialized lazily.
+ * @note Other than the `SparkContext`, all shared state is initialized lazily.
* This method will force the initialization of the shared state to ensure that parent
* and child sessions are set up with the same shared state. If the underlying catalog
* implementation is Hive, this will initialize the metastore, which may take some time.
@@ -219,7 +219,7 @@ class SparkSession private(
* --------------------------------- */
/**
- * Returns a [[DataFrame]] with no rows or columns.
+ * Returns a `DataFrame` with no rows or columns.
*
* @since 2.0.0
*/
@@ -243,7 +243,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Creates a [[DataFrame]] from an RDD of Product (e.g. case classes, tuples).
+ * Creates a `DataFrame` from an RDD of Product (e.g. case classes, tuples).
*
* @since 2.0.0
*/
@@ -257,7 +257,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Creates a [[DataFrame]] from a local Seq of Product.
+ * Creates a `DataFrame` from a local Seq of Product.
*
* @since 2.0.0
*/
@@ -272,7 +272,7 @@ class SparkSession private(
/**
* :: DeveloperApi ::
- * Creates a [[DataFrame]] from an [[RDD]] containing [[Row]]s using the given schema.
+ * Creates a `DataFrame` from an `RDD` containing [[Row]]s using the given schema.
* It is important to make sure that the structure of every [[Row]] of the provided RDD matches
* the provided schema. Otherwise, there will be runtime exception.
* Example:
@@ -309,7 +309,7 @@ class SparkSession private(
/**
* :: DeveloperApi ::
- * Creates a [[DataFrame]] from a [[JavaRDD]] containing [[Row]]s using the given schema.
+ * Creates a `DataFrame` from a `JavaRDD` containing [[Row]]s using the given schema.
* It is important to make sure that the structure of every [[Row]] of the provided RDD matches
* the provided schema. Otherwise, there will be runtime exception.
*
@@ -323,7 +323,7 @@ class SparkSession private(
/**
* :: DeveloperApi ::
- * Creates a [[DataFrame]] from a [[java.util.List]] containing [[Row]]s using the given schema.
+ * Creates a `DataFrame` from a [[java.util.List]] containing [[Row]]s using the given schema.
* It is important to make sure that the structure of every [[Row]] of the provided List matches
* the provided schema. Otherwise, there will be runtime exception.
*
@@ -381,7 +381,7 @@ class SparkSession private(
}
/**
- * Convert a [[BaseRelation]] created for external data sources into a [[DataFrame]].
+ * Convert a `BaseRelation` created for external data sources into a `DataFrame`.
*
* @since 2.0.0
*/
@@ -470,7 +470,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Creates a [[Dataset]] with a single [[LongType]] column named `id`, containing elements
+ * Creates a [[Dataset]] with a single `LongType` column named `id`, containing elements
* in a range from 0 to `end` (exclusive) with step value 1.
*
* @since 2.0.0
@@ -481,7 +481,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Creates a [[Dataset]] with a single [[LongType]] column named `id`, containing elements
+ * Creates a [[Dataset]] with a single `LongType` column named `id`, containing elements
* in a range from `start` to `end` (exclusive) with step value 1.
*
* @since 2.0.0
@@ -494,7 +494,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Creates a [[Dataset]] with a single [[LongType]] column named `id`, containing elements
+ * Creates a [[Dataset]] with a single `LongType` column named `id`, containing elements
* in a range from `start` to `end` (exclusive) with a step value.
*
* @since 2.0.0
@@ -507,7 +507,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Creates a [[Dataset]] with a single [[LongType]] column named `id`, containing elements
+ * Creates a [[Dataset]] with a single `LongType` column named `id`, containing elements
* in a range from `start` to `end` (exclusive) with a step value, with partition number
* specified.
*
@@ -520,7 +520,7 @@ class SparkSession private(
}
/**
- * Creates a [[DataFrame]] from an RDD[Row].
+ * Creates a `DataFrame` from an RDD[Row].
* User can specify whether the input rows should be converted to Catalyst rows.
*/
private[sql] def internalCreateDataFrame(
@@ -533,7 +533,7 @@ class SparkSession private(
}
/**
- * Creates a [[DataFrame]] from an RDD[Row].
+ * Creates a `DataFrame` from an RDD[Row].
* User can specify whether the input rows should be converted to Catalyst rows.
*/
private[sql] def createDataFrame(
@@ -566,7 +566,7 @@ class SparkSession private(
@transient lazy val catalog: Catalog = new CatalogImpl(self)
/**
- * Returns the specified table as a [[DataFrame]].
+ * Returns the specified table as a `DataFrame`.
*
* @since 2.0.0
*/
@@ -583,7 +583,7 @@ class SparkSession private(
* ----------------- */
/**
- * Executes a SQL query using Spark, returning the result as a [[DataFrame]].
+ * Executes a SQL query using Spark, returning the result as a `DataFrame`.
* The dialect that is used for SQL parsing can be configured with 'spark.sql.dialect'.
*
* @since 2.0.0
@@ -594,7 +594,7 @@ class SparkSession private(
/**
* Returns a [[DataFrameReader]] that can be used to read non-streaming data in as a
- * [[DataFrame]].
+ * `DataFrame`.
* {{{
* sparkSession.read.parquet("/path/to/file.parquet")
* sparkSession.read.schema(schema).json("/path/to/file.json")
@@ -606,7 +606,7 @@ class SparkSession private(
/**
* :: Experimental ::
- * Returns a [[DataStreamReader]] that can be used to read streaming data in as a [[DataFrame]].
+ * Returns a `DataStreamReader` that can be used to read streaming data in as a `DataFrame`.
* {{{
* sparkSession.readStream.parquet("/path/to/directory/of/parquet/files")
* sparkSession.readStream.schema(schema).json("/path/to/directory/of/json/files")
@@ -624,7 +624,7 @@ class SparkSession private(
/**
* :: Experimental ::
* (Scala-specific) Implicit methods available in Scala for converting
- * common Scala objects into [[DataFrame]]s.
+ * common Scala objects into `DataFrame`s.
*
* {{{
* val sparkSession = SparkSession.builder.getOrCreate()
@@ -641,7 +641,7 @@ class SparkSession private(
// scalastyle:on
/**
- * Stop the underlying [[SparkContext]].
+ * Stop the underlying `SparkContext`.
*
* @since 2.0.0
*/
@@ -726,7 +726,7 @@ object SparkSession {
/**
* Sets a config option. Options set using this method are automatically propagated to
- * both [[SparkConf]] and SparkSession's own configuration.
+ * both `SparkConf` and SparkSession's own configuration.
*
* @since 2.0.0
*/
@@ -737,7 +737,7 @@ object SparkSession {
/**
* Sets a config option. Options set using this method are automatically propagated to
- * both [[SparkConf]] and SparkSession's own configuration.
+ * both `SparkConf` and SparkSession's own configuration.
*
* @since 2.0.0
*/
@@ -748,7 +748,7 @@ object SparkSession {
/**
* Sets a config option. Options set using this method are automatically propagated to
- * both [[SparkConf]] and SparkSession's own configuration.
+ * both `SparkConf` and SparkSession's own configuration.
*
* @since 2.0.0
*/
@@ -759,7 +759,7 @@ object SparkSession {
/**
* Sets a config option. Options set using this method are automatically propagated to
- * both [[SparkConf]] and SparkSession's own configuration.
+ * both `SparkConf` and SparkSession's own configuration.
*
* @since 2.0.0
*/
@@ -769,7 +769,7 @@ object SparkSession {
}
/**
- * Sets a list of config options based on the given [[SparkConf]].
+ * Sets a list of config options based on the given `SparkConf`.
*
* @since 2.0.0
*/
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index 6043c5ee14b54e6e2f361f0c45f212f58a03e594..c8be89c64695725bcef646633e3b8cf326e283b4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -38,7 +38,7 @@ import org.apache.spark.sql.types.{DataType, DataTypes}
import org.apache.spark.util.Utils
/**
- * Functions for registering user-defined functions. Use [[SQLContext.udf]] to access this.
+ * Functions for registering user-defined functions. Use `SQLContext.udf` to access this.
*
* @note The user-defined functions must be deterministic.
*
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index d5940c638acdb2a0a2fdf5f8953870b26afa37c6..650439a19301555d06d94c0d0fee2093308a610c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -474,7 +474,9 @@ object functions {
/**
* Aggregate function: returns the level of grouping, equals to
*
- * (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)
+ * {{{
+ * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn)
+ * }}}
*
* @note The list of columns should match with grouping columns exactly, or empty (means all the
* grouping columns).
@@ -487,7 +489,9 @@ object functions {
/**
* Aggregate function: returns the level of grouping, equals to
*
- * (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)
+ * {{{
+ * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn)
+ * }}}
*
* @note The list of columns should match with grouping columns exactly.
*
@@ -1048,9 +1052,12 @@ object functions {
* within each partition in the lower 33 bits. The assumption is that the data frame has
* less than 1 billion partitions, and each partition has less than 8 billion records.
*
- * As an example, consider a [[DataFrame]] with two partitions, each with 3 records.
+ * As an example, consider a `DataFrame` with two partitions, each with 3 records.
* This expression would return the following IDs:
+ *
+ * {{{
* 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
+ * }}}
*
* @group normal_funcs
* @since 1.4.0
@@ -1066,9 +1073,12 @@ object functions {
* within each partition in the lower 33 bits. The assumption is that the data frame has
* less than 1 billion partitions, and each partition has less than 8 billion records.
*
- * As an example, consider a [[DataFrame]] with two partitions, each with 3 records.
+ * As an example, consider a `DataFrame` with two partitions, each with 3 records.
* This expression would return the following IDs:
+ *
+ * {{{
* 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
+ * }}}
*
* @group normal_funcs
* @since 1.6.0
@@ -1184,7 +1194,7 @@ object functions {
/**
* Creates a new struct column.
- * If the input column is a column in a [[DataFrame]], or a derived column expression
+ * If the input column is a column in a `DataFrame`, or a derived column expression
* that is named (i.e. aliased), its name would be remained as the StructField's name,
* otherwise, the newly generated StructField's name would be auto generated as col${index + 1},
* i.e. col1, col2, col3, ...
@@ -1846,8 +1856,8 @@ object functions {
def round(e: Column): Column = round(e, 0)
/**
- * Round the value of `e` to `scale` decimal places if `scale` >= 0
- * or at integral part when `scale` < 0.
+ * Round the value of `e` to `scale` decimal places if `scale` is greater than or equal to 0
+ * or at integral part when `scale` is less than 0.
*
* @group math_funcs
* @since 1.5.0
@@ -1864,7 +1874,7 @@ object functions {
/**
* Round the value of `e` to `scale` decimal places with HALF_EVEN round mode
- * if `scale` >= 0 or at integral part when `scale` < 0.
+ * if `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.
*
* @group math_funcs
* @since 2.0.0
@@ -2172,7 +2182,7 @@ object functions {
* and returns the result as a string column.
*
* If d is 0, the result has no decimal point or fractional part.
- * If d < 0, the result will be null.
+ * If d is less than 0, the result will be null.
*
* @group string_funcs
* @since 1.5.0
@@ -2888,7 +2898,7 @@ object functions {
}
/**
- * (Scala-specific) Parses a column containing a JSON string into a [[StructType]] with the
+ * (Scala-specific) Parses a column containing a JSON string into a `StructType` with the
* specified schema. Returns `null`, in the case of an unparseable string.
*
* @param e a string column containing JSON data.
@@ -2904,7 +2914,7 @@ object functions {
}
/**
- * (Java-specific) Parses a column containing a JSON string into a [[StructType]] with the
+ * (Java-specific) Parses a column containing a JSON string into a `StructType` with the
* specified schema. Returns `null`, in the case of an unparseable string.
*
* @param e a string column containing JSON data.
@@ -2919,7 +2929,7 @@ object functions {
from_json(e, schema, options.asScala.toMap)
/**
- * Parses a column containing a JSON string into a [[StructType]] with the specified schema.
+ * Parses a column containing a JSON string into a `StructType` with the specified schema.
* Returns `null`, in the case of an unparseable string.
*
* @param e a string column containing JSON data.
@@ -2932,7 +2942,7 @@ object functions {
from_json(e, schema, Map.empty[String, String])
/**
- * Parses a column containing a JSON string into a [[StructType]] with the specified schema.
+ * Parses a column containing a JSON string into a `StructType` with the specified schema.
* Returns `null`, in the case of an unparseable string.
*
* @param e a string column containing JSON data.
@@ -2946,7 +2956,7 @@ object functions {
/**
- * (Scala-specific) Converts a column containing a [[StructType]] into a JSON string with the
+ * (Scala-specific) Converts a column containing a `StructType` into a JSON string with the
* specified schema. Throws an exception, in the case of an unsupported type.
*
* @param e a struct column.
@@ -2961,7 +2971,7 @@ object functions {
}
/**
- * (Java-specific) Converts a column containing a [[StructType]] into a JSON string with the
+ * (Java-specific) Converts a column containing a `StructType` into a JSON string with the
* specified schema. Throws an exception, in the case of an unsupported type.
*
* @param e a struct column.
@@ -2975,7 +2985,7 @@ object functions {
to_json(e, options.asScala.toMap)
/**
- * Converts a column containing a [[StructType]] into a JSON string with the
+ * Converts a column containing a `StructType` into a JSON string with the
* specified schema. Throws an exception, in the case of an unsupported type.
*
* @param e a struct column.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index d3e323cb128914588bf465e1c9ae00ee07442a95..6d984621ccca1c90b1f54479357356bdfbc08ca2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.types.StructType
/**
- * Internal implementation of the user-facing [[Catalog]].
+ * Internal implementation of the user-facing `Catalog`.
*/
class CatalogImpl(sparkSession: SparkSession) extends Catalog {
@@ -175,8 +175,8 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
}
/**
- * Get the database with the specified name. This throws an [[AnalysisException]] when no
- * [[Database]] can be found.
+ * Get the database with the specified name. This throws an `AnalysisException` when no
+ * `Database` can be found.
*/
override def getDatabase(dbName: String): Database = {
makeDatabase(dbName)
@@ -184,7 +184,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
/**
* Get the table or view with the specified name. This table can be a temporary view or a
- * table/view in the current database. This throws an [[AnalysisException]] when no [[Table]]
+ * table/view in the current database. This throws an `AnalysisException` when no `Table`
* can be found.
*/
override def getTable(tableName: String): Table = {
@@ -193,7 +193,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
/**
* Get the table or view with the specified name in the specified database. This throws an
- * [[AnalysisException]] when no [[Table]] can be found.
+ * `AnalysisException` when no `Table` can be found.
*/
override def getTable(dbName: String, tableName: String): Table = {
makeTable(TableIdentifier(tableName, Option(dbName)))
@@ -201,7 +201,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
/**
* Get the function with the specified name. This function can be a temporary function or a
- * function in the current database. This throws an [[AnalysisException]] when no [[Function]]
+ * function in the current database. This throws an `AnalysisException` when no `Function`
* can be found.
*/
override def getFunction(functionName: String): Function = {
@@ -209,7 +209,7 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
}
/**
- * Get the function with the specified name. This returns [[None]] when no [[Function]] can be
+ * Get the function with the specified name. This returns `None` when no `Function` can be
* found.
*/
override def getFunction(dbName: String, functionName: String): Function = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala
index 791a9cf813b6a4f9e04938364b361e47efb94d11..4e7c813be9922e7aab32610ae18d95ac2464bf9c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/VariableSubstitution.scala
@@ -23,7 +23,7 @@ import org.apache.spark.internal.config._
* A helper class that enables substitution using syntax like
* `${var}`, `${system:var}` and `${env:var}`.
*
- * Variable substitution is controlled by [[SQLConf.variableSubstituteEnabled]].
+ * Variable substitution is controlled by `SQLConf.variableSubstituteEnabled`.
*/
class VariableSubstitution(conf: SQLConf) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
index 83857c322a0ecf1abbaa943085ee22c07e5faf16..e328b86437d62a6a93f8f7deafc4294653fadbc3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
@@ -40,8 +40,8 @@ case class JdbcType(databaseTypeDefinition : String, jdbcNullType : Int)
* SQL dialect of a certain database or jdbc driver.
* Lots of databases define types that aren't explicitly supported
* by the JDBC spec. Some JDBC drivers also report inaccurate
- * information---for instance, BIT(n>1) being reported as a BIT type is quite
- * common, even though BIT in JDBC is meant for single-bit values. Also, there
+ * information---for instance, BIT(n{@literal >}1) being reported as a BIT type is quite
+ * common, even though BIT in JDBC is meant for single-bit values. Also, there
* does not appear to be a standard name for an unbounded string or binary
* type; we use BLOB and CLOB by default but override with database-specific
* alternatives when these are absent or do not behave correctly.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
index ff6dd8cb0cf92c82b205358cf266a9cd36dae4e5..f288ad61410f7cd97060a2fb0376b88ea38cd1fc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
@@ -112,7 +112,7 @@ trait SchemaRelationProvider {
/**
* ::Experimental::
- * Implemented by objects that can produce a streaming [[Source]] for a specific format or system.
+ * Implemented by objects that can produce a streaming `Source` for a specific format or system.
*
* @since 2.0.0
*/
@@ -143,7 +143,7 @@ trait StreamSourceProvider {
/**
* ::Experimental::
- * Implemented by objects that can produce a streaming [[Sink]] for a specific format or system.
+ * Implemented by objects that can produce a streaming `Sink` for a specific format or system.
*
* @since 2.0.0
*/
@@ -185,7 +185,7 @@ trait CreatableRelationProvider {
/**
* Represents a collection of tuples with a known schema. Classes that extend BaseRelation must
- * be able to produce the schema of their data in the form of a [[StructType]]. Concrete
+ * be able to produce the schema of their data in the form of a `StructType`. Concrete
* implementation should inherit from one of the descendant `Scan` classes, which define various
* abstract methods for execution.
*
@@ -216,10 +216,10 @@ abstract class BaseRelation {
/**
* Whether does it need to convert the objects in Row to internal representation, for example:
- * java.lang.String -> UTF8String
- * java.lang.Decimal -> Decimal
+ * java.lang.String to UTF8String
+ * java.lang.Decimal to Decimal
*
- * If `needConversion` is `false`, buildScan() should return an [[RDD]] of [[InternalRow]]
+ * If `needConversion` is `false`, buildScan() should return an `RDD` of `InternalRow`
*
* @note The internal representation is not stable across releases and thus data sources outside
* of Spark SQL should leave this as true.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index a2d64da0012f148d8f019f149711f463ec0cb9a1..5f5c8e2432d6cc5ed3f89b35a279758268f1ac97 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -57,9 +57,17 @@ import org.apache.spark.util.SerializableJobConf
* @param partition a map from the partition key to the partition value (optional). If the partition
* value is optional, dynamic partition insert will be performed.
* As an example, `INSERT INTO tbl PARTITION (a=1, b=2) AS ...` would have
- * Map('a' -> Some('1'), 'b' -> Some('2')),
+ *
+ * {{{
+ * Map('a' -> Some('1'), 'b' -> Some('2'))
+ * }}}
+ *
* and `INSERT INTO tbl PARTITION (a=1, b) AS ...`
- * would have Map('a' -> Some('1'), 'b' -> None).
+ * would have
+ *
+ * {{{
+ * Map('a' -> Some('1'), 'b' -> None)
+ * }}}.
* @param child the logical plan representing data to write to.
* @param overwrite overwrite existing table or partitions.
* @param ifNotExists If true, only write if the table or partition does not exist.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 32edd4aec2865e131381599c452b86bff7f2434e..90e86959cd0e47fd864316e659c8d9454410eafb 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -177,7 +177,7 @@ private[hive] case class HiveGenericUDF(
/**
* Converts a Hive Generic User Defined Table Generating Function (UDTF) to a
- * [[Generator]]. Note that the semantics of Generators do not allow
+ * `Generator`. Note that the semantics of Generators do not allow
* Generators to maintain state in between input rows. Thus UDTFs that rely on partitioning
* dependent operations like calls to `close()` before producing output will not operate the same as
* in Hive. However, in practice this should not affect compatibility for most sane UDTFs
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala
index a34e2e76f5838936aaa7c16a7e66a49147588c10..0c9321068c4c1cc2561bd552b9a7294174b3af37 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveWriterContainers.scala
@@ -49,7 +49,7 @@ import org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter
/**
* Internal helper class that saves an RDD using a Hive OutputFormat.
- * It is based on [[SparkHadoopWriter]].
+ * It is based on `SparkHadoopWriter`.
*/
private[hive] class SparkHiveWriterContainer(
@transient private val jobConf: JobConf,
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala b/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala
index 7c1ea2f89ddb8f3c2a08c0a4ee847db89e9497e6..c3b28bd516da591ac6f83038f3f91e07562e7bde 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StateSpec.scala
@@ -30,7 +30,7 @@ import org.apache.spark.util.ClosureCleaner
* `mapWithState` operation of a
* [[org.apache.spark.streaming.dstream.PairDStreamFunctions pair DStream]] (Scala) or a
* [[org.apache.spark.streaming.api.java.JavaPairDStream JavaPairDStream]] (Java).
- * Use [[org.apache.spark.streaming.StateSpec.function() StateSpec.function]] factory methods
+ * Use `org.apache.spark.streaming.StateSpec.function()` factory methods
* to create instances of this class.
*
* Example in Scala:
@@ -100,7 +100,7 @@ sealed abstract class StateSpec[KeyType, ValueType, StateType, MappedType] exten
/**
* :: Experimental ::
- * Builder object for creating instances of [[org.apache.spark.streaming.StateSpec StateSpec]]
+ * Builder object for creating instances of `org.apache.spark.streaming.StateSpec`
* that is used for specifying the parameters of the DStream transformation `mapWithState`
* that is used for specifying the parameters of the DStream transformation
* `mapWithState` operation of a
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala
index aa4003c62e1e7aa179aaae20d93dd0ed1836341c..2ec907c8cfd5f11c2fb2e73ce2f45ce17ffda5c2 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala
@@ -434,8 +434,8 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
* Return a [[JavaMapWithStateDStream]] by applying a function to every key-value element of
* `this` stream, while maintaining some state data for each unique key. The mapping function
* and other specification (e.g. partitioners, timeouts, initial state data, etc.) of this
- * transformation can be specified using [[StateSpec]] class. The state data is accessible in
- * as a parameter of type [[State]] in the mapping function.
+ * transformation can be specified using `StateSpec` class. The state data is accessible in
+ * as a parameter of type `State` in the mapping function.
*
* Example of using `mapWithState`:
* {{{
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
index b43b9405def971f904d83be09209289e4d61e5e3..982e72cffbf3fa1a051008f82ba73600ce6d334c 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
@@ -44,7 +44,7 @@ import org.apache.spark.streaming.scheduler.StreamingListener
* A Java-friendly version of [[org.apache.spark.streaming.StreamingContext]] which is the main
* entry point for Spark Streaming functionality. It provides methods to create
* [[org.apache.spark.streaming.api.java.JavaDStream]] and
- * [[org.apache.spark.streaming.api.java.JavaPairDStream.]] from input sources. The internal
+ * [[org.apache.spark.streaming.api.java.JavaPairDStream]] from input sources. The internal
* org.apache.spark.api.java.JavaSparkContext (see core Spark documentation) can be accessed
* using `context.sparkContext`. After creating and transforming DStreams, the streaming
* computation can be started and stopped using `context.start()` and `context.stop()`,
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.scala
index ac739411fd2120d471191ee3fcda960c2ffeede9..f38c1e799659520ca43630587b61d7904122632b 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.scala
@@ -356,8 +356,8 @@ class PairDStreamFunctions[K, V](self: DStream[(K, V)])
* Return a [[MapWithStateDStream]] by applying a function to every key-value element of
* `this` stream, while maintaining some state data for each unique key. The mapping function
* and other specification (e.g. partitioners, timeouts, initial state data, etc.) of this
- * transformation can be specified using [[StateSpec]] class. The state data is accessible in
- * as a parameter of type [[State]] in the mapping function.
+ * transformation can be specified using `StateSpec` class. The state data is accessible in
+ * as a parameter of type `State` in the mapping function.
*
* Example of using `mapWithState`:
* {{{