Skip to content
Snippets Groups Projects
Commit a277ae80 authored by Michal Szafranski's avatar Michal Szafranski Committed by Reynold Xin
Browse files

[SPARK-20474] Fixing OnHeapColumnVector reallocation

## What changes were proposed in this pull request?
OnHeapColumnVector reallocation copies to the new storage data up to 'elementsAppended'. This variable is only updated when using the ColumnVector.appendX API, while ColumnVector.putX is more commonly used.

## How was this patch tested?
Tested using existing unit tests.

Author: Michal Szafranski <michal@databricks.com>

Closes #17773 from michal-databricks/spark-20474.
parent 99c6cf9e
No related branches found
No related tags found
No related merge requests found
......@@ -410,53 +410,53 @@ public final class OnHeapColumnVector extends ColumnVector {
int[] newLengths = new int[newCapacity];
int[] newOffsets = new int[newCapacity];
if (this.arrayLengths != null) {
System.arraycopy(this.arrayLengths, 0, newLengths, 0, elementsAppended);
System.arraycopy(this.arrayOffsets, 0, newOffsets, 0, elementsAppended);
System.arraycopy(this.arrayLengths, 0, newLengths, 0, capacity);
System.arraycopy(this.arrayOffsets, 0, newOffsets, 0, capacity);
}
arrayLengths = newLengths;
arrayOffsets = newOffsets;
} else if (type instanceof BooleanType) {
if (byteData == null || byteData.length < newCapacity) {
byte[] newData = new byte[newCapacity];
if (byteData != null) System.arraycopy(byteData, 0, newData, 0, elementsAppended);
if (byteData != null) System.arraycopy(byteData, 0, newData, 0, capacity);
byteData = newData;
}
} else if (type instanceof ByteType) {
if (byteData == null || byteData.length < newCapacity) {
byte[] newData = new byte[newCapacity];
if (byteData != null) System.arraycopy(byteData, 0, newData, 0, elementsAppended);
if (byteData != null) System.arraycopy(byteData, 0, newData, 0, capacity);
byteData = newData;
}
} else if (type instanceof ShortType) {
if (shortData == null || shortData.length < newCapacity) {
short[] newData = new short[newCapacity];
if (shortData != null) System.arraycopy(shortData, 0, newData, 0, elementsAppended);
if (shortData != null) System.arraycopy(shortData, 0, newData, 0, capacity);
shortData = newData;
}
} else if (type instanceof IntegerType || type instanceof DateType ||
DecimalType.is32BitDecimalType(type)) {
if (intData == null || intData.length < newCapacity) {
int[] newData = new int[newCapacity];
if (intData != null) System.arraycopy(intData, 0, newData, 0, elementsAppended);
if (intData != null) System.arraycopy(intData, 0, newData, 0, capacity);
intData = newData;
}
} else if (type instanceof LongType || type instanceof TimestampType ||
DecimalType.is64BitDecimalType(type)) {
if (longData == null || longData.length < newCapacity) {
long[] newData = new long[newCapacity];
if (longData != null) System.arraycopy(longData, 0, newData, 0, elementsAppended);
if (longData != null) System.arraycopy(longData, 0, newData, 0, capacity);
longData = newData;
}
} else if (type instanceof FloatType) {
if (floatData == null || floatData.length < newCapacity) {
float[] newData = new float[newCapacity];
if (floatData != null) System.arraycopy(floatData, 0, newData, 0, elementsAppended);
if (floatData != null) System.arraycopy(floatData, 0, newData, 0, capacity);
floatData = newData;
}
} else if (type instanceof DoubleType) {
if (doubleData == null || doubleData.length < newCapacity) {
double[] newData = new double[newCapacity];
if (doubleData != null) System.arraycopy(doubleData, 0, newData, 0, elementsAppended);
if (doubleData != null) System.arraycopy(doubleData, 0, newData, 0, capacity);
doubleData = newData;
}
} else if (resultStruct != null) {
......@@ -466,7 +466,7 @@ public final class OnHeapColumnVector extends ColumnVector {
}
byte[] newNulls = new byte[newCapacity];
if (nulls != null) System.arraycopy(nulls, 0, newNulls, 0, elementsAppended);
if (nulls != null) System.arraycopy(nulls, 0, newNulls, 0, capacity);
nulls = newNulls;
capacity = newCapacity;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment