diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index f904d810c6c..78a049ce12a 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -118,7 +118,7 @@ pub fn vortex_alp::ALPArray::into_parts(self) -> (vortex_array::array::ArrayRef, pub fn vortex_alp::ALPArray::new(encoded: vortex_array::array::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> Self -pub fn vortex_alp::ALPArray::patches(&self) -> core::option::Option<&vortex_array::patches::Patches> +pub fn vortex_alp::ALPArray::patches(&self) -> core::option::Option pub fn vortex_alp::ALPArray::ptype(&self) -> vortex_array::dtype::ptype::PType @@ -278,7 +278,7 @@ pub fn vortex_alp::ALPRDArray::left_parts(&self) -> &vortex_array::array::ArrayR pub fn vortex_alp::ALPRDArray::left_parts_dictionary(&self) -> &vortex_buffer::buffer::Buffer -pub fn vortex_alp::ALPRDArray::left_parts_patches(&self) -> core::option::Option<&vortex_array::patches::Patches> +pub fn vortex_alp::ALPRDArray::left_parts_patches(&self) -> core::option::Option pub fn vortex_alp::ALPRDArray::replace_left_parts_patches(&mut self, patches: core::option::Option) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index b3858a3d26a..dc0e15879e7 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -76,14 +76,14 @@ impl VTable for ALP { array.dtype.hash(state); array.encoded().array_hash(state, precision); array.exponents.hash(state); - array.patches.array_hash(state, precision); + array.patches().array_hash(state, precision); } fn array_eq(array: &ALPArray, other: &ALPArray, precision: Precision) -> bool { array.dtype == other.dtype && array.encoded().array_eq(other.encoded(), precision) && array.exponents == other.exponents - && array.patches.array_eq(&other.patches, precision) + && array.patches().array_eq(&other.patches(), precision) } fn nbuffers(_array: &ALPArray) -> usize { @@ -114,23 +114,12 @@ impl VTable for ALP { slots.len() ); - // Reconstruct patches from slots + existing metadata - array.patches = match (&slots[PATCH_INDICES_SLOT], &slots[PATCH_VALUES_SLOT]) { - (Some(indices), Some(values)) => { - let old = array - .patches - .as_ref() - .vortex_expect("ALPArray had patch slots but no patches metadata"); - Some(Patches::new( - old.array_len(), - old.offset(), - indices.clone(), - values.clone(), - slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), - )?) - } - _ => None, - }; + // If patch slots are being cleared, clear the metadata too + if slots[PATCH_INDICES_SLOT].is_none() || slots[PATCH_VALUES_SLOT].is_none() { + array.patch_offset = None; + array.patch_offset_within_chunk = None; + } + array.slots = slots; Ok(()) } @@ -240,7 +229,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ #[derive(Clone, Debug)] pub struct ALPArray { slots: Vec>, - patches: Option, + patch_offset: Option, + patch_offset_within_chunk: Option, dtype: DType, exponents: Exponents, stats_set: ArrayStats, @@ -409,12 +399,17 @@ impl ALPArray { }; let slots = Self::make_slots(&encoded, &patches); + let (patch_offset, patch_offset_within_chunk) = match &patches { + Some(p) => (Some(p.offset()), p.offset_within_chunk()), + None => (None, None), + }; Ok(Self { dtype, slots, exponents, - patches, + patch_offset, + patch_offset_within_chunk, stats_set: Default::default(), }) } @@ -430,12 +425,17 @@ impl ALPArray { dtype: DType, ) -> Self { let slots = Self::make_slots(&encoded, &patches); + let (patch_offset, patch_offset_within_chunk) = match &patches { + Some(p) => (Some(p.offset()), p.offset_within_chunk()), + None => (None, None), + }; Self { dtype, slots, exponents, - patches, + patch_offset, + patch_offset_within_chunk, stats_set: Default::default(), } } @@ -472,17 +472,38 @@ impl ALPArray { self.exponents } - pub fn patches(&self) -> Option<&Patches> { - self.patches.as_ref() + pub fn patches(&self) -> Option { + match ( + &self.slots[PATCH_INDICES_SLOT], + &self.slots[PATCH_VALUES_SLOT], + ) { + (Some(indices), Some(values)) => { + let patch_offset = self + .patch_offset + .vortex_expect("has patch slots but no patch_offset"); + Some(unsafe { + Patches::new_unchecked( + self.encoded().len(), + patch_offset, + indices.clone(), + values.clone(), + self.slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + self.patch_offset_within_chunk, + ) + }) + } + _ => None, + } } /// Consumes the array and returns its parts. #[inline] pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option, DType) { + let patches = self.patches(); let encoded = self.slots[ENCODED_SLOT] .take() .vortex_expect("ALPArray encoded slot"); - (encoded, self.exponents, self.patches, self.dtype) + (encoded, self.exponents, patches, self.dtype) } } @@ -506,7 +527,6 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::session::ArraySession; - use vortex_array::vtable::ValidityHelper; use vortex_session::VortexSession; use super::*; diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 3759f354a22..aaea9563fa0 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -8,7 +8,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::PType; use vortex_array::patches::Patches; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; @@ -73,7 +72,7 @@ where let (exponents, encoded, exceptional_positions, exceptional_values, mut chunk_offsets) = T::encode(values_slice, exponents); - let encoded_array = PrimitiveArray::new(encoded, values.validity().clone()).into_array(); + let encoded_array = PrimitiveArray::new(encoded, values.validity()).into_array(); let validity = values.validity_mask()?; // exceptional_positions may contain exceptions at invalid positions (which contain garbage diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index 813f195c3b8..3185c69d66c 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -29,7 +29,7 @@ impl CastReduce for ALP { .patches() .map(|p| { if p.values().dtype() == dtype { - Ok(p.clone()) + Ok(p) } else { Patches::new( p.array_len(), diff --git a/encodings/alp/src/alp/decompress.rs b/encodings/alp/src/alp/decompress.rs index d2a921dfaed..7320cddf1db 100644 --- a/encodings/alp/src/alp/decompress.rs +++ b/encodings/alp/src/alp/decompress.rs @@ -10,7 +10,6 @@ use vortex_array::arrays::primitive::patch_chunk; use vortex_array::dtype::DType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::BufferMut; use vortex_error::VortexResult; @@ -101,7 +100,7 @@ fn decompress_chunked_core( patches: &Patches, dtype: DType, ) -> PrimitiveArray { - let validity = encoded.validity().clone(); + let validity = encoded.validity(); let ptype = dtype.as_ptype(); let array_len = encoded.len(); let offset_within_chunk = patches.offset_within_chunk().unwrap_or(0); @@ -151,7 +150,7 @@ fn decompress_unchunked_core( dtype: DType, ctx: &mut ExecutionCtx, ) -> VortexResult { - let validity = encoded.validity().clone(); + let validity = encoded.validity(); let ptype = dtype.as_ptype(); let decoded = match_each_alp_float_ptype!(ptype, |T| { diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index fe7385e9299..5db84afd3a6 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -543,8 +543,8 @@ impl ALPRDArray { } /// Patches of left-most bits. - pub fn left_parts_patches(&self) -> Option<&Patches> { - self.left_parts_patches.as_ref() + pub fn left_parts_patches(&self) -> Option { + self.left_parts_patches.clone() } /// The dictionary that maps the codes in `left_parts` into bit patterns. diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 9a2b4d73ec2..a1441d6e416 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -34,7 +34,7 @@ impl CastReduce for ALPRD { array.left_parts_dictionary().clone(), array.right_parts().clone(), array.right_bit_width(), - array.left_parts_patches().cloned(), + array.left_parts_patches(), )? .into_array(), )); diff --git a/encodings/alp/src/alp_rd/compute/mask.rs b/encodings/alp/src/alp_rd/compute/mask.rs index 4a6ad8a7b6a..6126eeef301 100644 --- a/encodings/alp/src/alp_rd/compute/mask.rs +++ b/encodings/alp/src/alp_rd/compute/mask.rs @@ -26,7 +26,7 @@ impl MaskReduce for ALPRD { array.left_parts_dictionary().clone(), array.right_parts().clone(), array.right_bit_width(), - array.left_parts_patches().cloned(), + array.left_parts_patches(), )? .into_array(), )) diff --git a/encodings/alp/src/alp_rd/mod.rs b/encodings/alp/src/alp_rd/mod.rs index 7521ff15b7c..a7cefe3c35d 100644 --- a/encodings/alp/src/alp_rd/mod.rs +++ b/encodings/alp/src/alp_rd/mod.rs @@ -30,7 +30,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::match_each_integer_ptype; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; @@ -229,7 +228,7 @@ impl RDEncoder { } // Bit-pack down the encoded left-parts array that have been dictionary encoded. - let primitive_left = PrimitiveArray::new(left_parts, array.validity().clone()); + let primitive_left = PrimitiveArray::new(left_parts, array.validity()); // SAFETY: by construction, all values in left_parts can be packed to left_bit_width. let packed_left = unsafe { bitpack_encode_unchecked(primitive_left, left_bit_width as _) diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index fcbc96d13d9..dc9c43f62b8 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -136,4 +136,4 @@ pub fn vortex_bytebool::ByteBoolArray::into_array(self) -> vortex_array::array:: impl vortex_array::vtable::validity::ValidityHelper for vortex_bytebool::ByteBoolArray -pub fn vortex_bytebool::ByteBoolArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_bytebool::ByteBoolArray::validity(&self) -> vortex_array::validity::Validity diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index b25e06fdb28..2badc87933e 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -177,7 +177,7 @@ impl VTable for ByteBool { fn execute(array: Arc>, _ctx: &mut ExecutionCtx) -> VortexResult { let boolean_buffer = BitBuffer::from(array.as_slice()); - let validity = array.validity().clone(); + let validity = array.validity(); Ok(ExecutionResult::done( BoolArray::new(boolean_buffer, validity).into_array(), )) @@ -258,8 +258,8 @@ impl ByteBoolArray { } impl ValidityHelper for ByteBoolArray { - fn validity(&self) -> &Validity { - &self.validity + fn validity(&self) -> Validity { + self.validity.clone() } } diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index b0583961638..ac7e6a23b61 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -28,7 +28,6 @@ impl CastReduce for ByteBool { if array.dtype().eq_ignore_nullability(dtype) { let new_validity = array .validity() - .clone() .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( @@ -46,10 +45,7 @@ impl MaskReduce for ByteBool { Ok(Some( ByteBoolArray::new( array.buffer().clone(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .into_array(), )) diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index f619dd70a40..96d81c9b477 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -115,7 +115,6 @@ mod test { use vortex_array::assert_arrays_eq; use vortex_array::extension::datetime::TimeUnit; use vortex_array::validity::Validity; - use vortex_array::vtable::ValidityHelper; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; diff --git a/encodings/datetime-parts/src/compress.rs b/encodings/datetime-parts/src/compress.rs index 6010f6adeaf..10442e27954 100644 --- a/encodings/datetime-parts/src/compress.rs +++ b/encodings/datetime-parts/src/compress.rs @@ -9,7 +9,6 @@ use vortex_array::arrays::TemporalArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::BufferMut; use vortex_error::VortexError; use vortex_error::VortexResult; @@ -53,7 +52,7 @@ pub fn split_temporal(array: TemporalArray) -> VortexResult { } Ok(TemporalParts { - days: PrimitiveArray::new(days, temporal_values.validity().clone()).into_array(), + days: PrimitiveArray::new(days, temporal_values.validity()).into_array(), seconds: seconds.into_array(), subseconds: subseconds.into_array(), }) @@ -84,7 +83,6 @@ mod tests { use vortex_array::arrays::TemporalArray; use vortex_array::extension::datetime::TimeUnit; use vortex_array::validity::Validity; - use vortex_array::vtable::ValidityHelper; use vortex_buffer::buffer; use crate::TemporalParts; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 309485e3978..fdc6ccddb4c 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -38,7 +38,6 @@ use vortex_array::vtable::ArrayId; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; -use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValidityVTableFromChild; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -302,7 +301,7 @@ fn to_canonical_decimal( DecimalArray::new_unchecked( prim.to_buffer::

(), *array.decimal_dtype(), - prim.validity().clone(), + prim.validity(), ) } .into_array() diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 2381a4a9b6f..e6638186cfd 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -154,7 +154,7 @@ pub type vortex_fastlanes::BitPacked::Metadata = vortex_array::metadata::ProstMe pub type vortex_fastlanes::BitPacked::OperationsVTable = vortex_fastlanes::BitPacked -pub type vortex_fastlanes::BitPacked::ValidityVTable = vortex_array::vtable::validity::ValidityVTableFromValidityHelper +pub type vortex_fastlanes::BitPacked::ValidityVTable = vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::append_to_builder(array: &vortex_fastlanes::BitPackedArray, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -202,6 +202,10 @@ impl vortex_array::vtable::operations::OperationsVTable vortex_error::VortexResult +impl vortex_array::vtable::validity::ValidityVTable for vortex_fastlanes::BitPacked + +pub fn vortex_fastlanes::BitPacked::validity(array: &vortex_fastlanes::BitPackedArray) -> vortex_error::VortexResult + pub struct vortex_fastlanes::BitPackedArray impl vortex_fastlanes::BitPackedArray @@ -220,7 +224,7 @@ pub fn vortex_fastlanes::BitPackedArray::packed(&self) -> &vortex_array::buffer: pub fn vortex_fastlanes::BitPackedArray::packed_slice(&self) -> &[T] -pub fn vortex_fastlanes::BitPackedArray::patches(&self) -> core::option::Option<&vortex_array::patches::Patches> +pub fn vortex_fastlanes::BitPackedArray::patches(&self) -> core::option::Option pub fn vortex_fastlanes::BitPackedArray::ptype(&self) -> vortex_array::dtype::ptype::PType @@ -230,6 +234,8 @@ pub fn vortex_fastlanes::BitPackedArray::try_new(packed: vortex_array::buffer::B pub fn vortex_fastlanes::BitPackedArray::unpacked_chunks(&self) -> vortex_fastlanes::unpack_iter::BitUnpackedChunks +pub fn vortex_fastlanes::BitPackedArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_fastlanes::BitPackedArray pub fn vortex_fastlanes::BitPackedArray::to_array(&self) -> vortex_array::array::ArrayRef @@ -260,10 +266,6 @@ impl vortex_array::array::IntoArray for vortex_fastlanes::BitPackedArray pub fn vortex_fastlanes::BitPackedArray::into_array(self) -> vortex_array::array::ArrayRef -impl vortex_array::vtable::validity::ValidityHelper for vortex_fastlanes::BitPackedArray - -pub fn vortex_fastlanes::BitPackedArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_fastlanes::BitPackedArrayParts pub vortex_fastlanes::BitPackedArrayParts::bit_width: u8 diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 6f29a72db0c..e56f39633f5 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -14,7 +14,6 @@ use vortex_array::match_each_integer_ptype; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_buffer::ByteBuffer; @@ -76,7 +75,7 @@ pub fn bitpack_encode( BitPackedArray::new_unchecked( BufferHandle::new_host(packed), array.dtype().clone(), - array.validity().clone(), + array.validity(), patches, bit_width, array.len(), @@ -110,7 +109,7 @@ pub unsafe fn bitpack_encode_unchecked( BitPackedArray::new_unchecked( BufferHandle::new_host(packed), array.dtype().clone(), - array.validity().clone(), + array.validity(), None, bit_width, array.len(), diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index e4099cdcf24..372ac81af52 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -65,7 +65,7 @@ pub(crate) fn unpack_into_primitive_builder( let mut bit_packed_iter = array.unpacked_chunks(); bit_packed_iter.decode_into(uninit_slice); - if let Some(patches) = array.patches() { + if let Some(ref patches) = array.patches() { apply_patches_to_uninit_range(&mut uninit_range, patches, ctx)?; }; diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 6113f815148..67c4b0c09ed 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -11,7 +11,9 @@ use vortex_array::dtype::PType; use vortex_array::patches::Patches; use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; +use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -55,8 +57,10 @@ pub struct BitPackedArray { pub(super) dtype: DType, pub(super) bit_width: u8, pub(super) packed: BufferHandle, - pub(super) patches: Option, - pub(super) validity: Validity, + /// The offset metadata from patches, needed to reconstruct Patches from slots. + pub(super) patch_offset: Option, + /// The offset_within_chunk metadata from patches. + pub(super) patch_offset_within_chunk: Option, pub(super) stats_set: ArrayStats, } @@ -91,6 +95,10 @@ impl BitPackedArray { offset: u16, ) -> Self { let slots = Self::make_slots(&patches, &validity, len); + let (patch_offset, patch_offset_within_chunk) = match &patches { + Some(p) => (Some(p.offset()), p.offset_within_chunk()), + None => (None, None), + }; Self { slots, @@ -99,8 +107,8 @@ impl BitPackedArray { dtype, bit_width, packed, - patches, - validity, + patch_offset, + patch_offset_within_chunk, stats_set: Default::default(), } } @@ -275,15 +283,39 @@ impl BitPackedArray { /// Access the patches array. /// + /// Reconstructs a `Patches` from the stored slots and patch metadata. /// If present, patches MUST be a `SparseArray` with equal-length to this array, and whose /// indices indicate the locations of patches. The indices must have non-zero length. - #[inline] - pub fn patches(&self) -> Option<&Patches> { - self.patches.as_ref() + pub fn patches(&self) -> Option { + match ( + &self.slots[PATCH_INDICES_SLOT], + &self.slots[PATCH_VALUES_SLOT], + ) { + (Some(indices), Some(values)) => { + let patch_offset = self + .patch_offset + .vortex_expect("has patch slots but no patch_offset"); + Some(unsafe { + Patches::new_unchecked( + self.len, + patch_offset, + indices.clone(), + values.clone(), + self.slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), + self.patch_offset_within_chunk, + ) + }) + } + _ => None, + } + } + + /// Returns the validity, reconstructed from the stored slot. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } pub fn replace_patches(&mut self, patches: Option) { - // Update both the patches and the corresponding slots to keep them in sync. let (pi, pv, pco) = match &patches { Some(p) => ( Some(p.indices().clone()), @@ -295,7 +327,8 @@ impl BitPackedArray { self.slots[PATCH_INDICES_SLOT] = pi; self.slots[PATCH_VALUES_SLOT] = pv; self.slots[PATCH_CHUNK_OFFSETS_SLOT] = pco; - self.patches = patches; + self.patch_offset = patches.as_ref().map(|p| p.offset()); + self.patch_offset_within_chunk = patches.as_ref().and_then(|p| p.offset_within_chunk()); } #[inline] @@ -332,13 +365,15 @@ impl BitPackedArray { } pub fn into_parts(self) -> BitPackedArrayParts { + let patches = self.patches(); + let validity = self.validity(); BitPackedArrayParts { offset: self.offset, bit_width: self.bit_width, len: self.len, packed: self.packed, - patches: self.patches, - validity: self.validity, + patches, + validity, } } } diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index b6ba46626d3..1480f24a18f 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -7,7 +7,6 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::patches::Patches; use vortex_array::scalar_fn::fns::cast::CastReduce; -use vortex_array::vtable::ValidityHelper; use vortex_error::VortexResult; use crate::bitpacking::BitPacked; @@ -18,7 +17,6 @@ impl CastReduce for BitPacked { if array.dtype().eq_ignore_nullability(dtype) { let new_validity = array .validity() - .clone() .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( BitPackedArray::try_new( diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index f394f76a26f..69452e02568 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -98,7 +98,7 @@ fn filter_primitive_without_patches( selection: &Arc, ) -> VortexResult<(Buffer, Validity)> { let values = filter_with_indices(array, selection.indices()); - let validity = array.validity()?.filter(&Mask::Values(selection.clone()))?; + let validity = array.validity().filter(&Mask::Values(selection.clone()))?; Ok((values.freeze(), validity)) } diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index 3219637b8fc..4449cdb01f3 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -29,7 +29,7 @@ impl SliceReduce for BitPacked { BitPackedArray::new_unchecked( array.packed().slice(encoded_start..encoded_stop), array.dtype().clone(), - array.validity()?.slice(range.clone())?, + array.validity().slice(range.clone())?, array .patches() .map(|p| p.slice(range.clone())) diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 4405645ace3..9e9289ce133 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -17,7 +17,6 @@ use vortex_array::dtype::PType; use vortex_array::match_each_integer_ptype; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect as _; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index ef8d3aa8651..138c69acf79 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -28,7 +28,6 @@ use vortex_array::vtable; use vortex_array::vtable::Array; use vortex_array::vtable::ArrayId; use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTableFromValidityHelper; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -41,11 +40,9 @@ use crate::BitPackedArray; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; use crate::bitpacking::array::NUM_SLOTS; -use crate::bitpacking::array::PATCH_CHUNK_OFFSETS_SLOT; use crate::bitpacking::array::PATCH_INDICES_SLOT; use crate::bitpacking::array::PATCH_VALUES_SLOT; use crate::bitpacking::array::SLOT_NAMES; -use crate::bitpacking::array::VALIDITY_SLOT; use crate::bitpacking::vtable::kernels::PARENT_KERNELS; use crate::bitpacking::vtable::rules::RULES; mod kernels; @@ -71,7 +68,7 @@ impl VTable for BitPacked { type Metadata = ProstMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &BitPacked @@ -103,8 +100,8 @@ impl VTable for BitPacked { array.dtype.hash(state); array.bit_width.hash(state); array.packed.array_hash(state, precision); - array.patches.array_hash(state, precision); - array.validity.array_hash(state, precision); + array.patches().array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &BitPackedArray, other: &BitPackedArray, precision: Precision) -> bool { @@ -113,8 +110,8 @@ impl VTable for BitPacked { && array.dtype == other.dtype && array.bit_width == other.bit_width && array.packed.array_eq(&other.packed, precision) - && array.patches.array_eq(&other.patches, precision) - && array.validity.array_eq(&other.validity, precision) + && array.patches().array_eq(&other.patches(), precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: &BitPackedArray) -> usize { @@ -159,32 +156,11 @@ impl VTable for BitPacked { slots.len() ); - // Reconstruct patches from slots + existing metadata - array.patches = match (&slots[PATCH_INDICES_SLOT], &slots[PATCH_VALUES_SLOT]) { - (Some(indices), Some(values)) => { - let old = array - .patches - .as_ref() - .vortex_expect("BitPackedArray had patch slots but no patches metadata"); - Some(unsafe { - Patches::new_unchecked( - array.len, - old.offset(), - indices.clone(), - values.clone(), - slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), - None, - ) - }) - } - _ => None, - }; - - // Reconstruct validity from slot - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), - }; + // If patch slots are being cleared, clear the metadata too + if slots[PATCH_INDICES_SLOT].is_none() || slots[PATCH_VALUES_SLOT].is_none() { + array.patch_offset = None; + array.patch_offset_within_chunk = None; + } array.slots = slots; Ok(()) diff --git a/encodings/fastlanes/src/bitpacking/vtable/validity.rs b/encodings/fastlanes/src/bitpacking/vtable/validity.rs index feafa6fbc44..64e33cb23cc 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/validity.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/validity.rs @@ -2,12 +2,14 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; +use vortex_array::vtable::ValidityVTable; +use vortex_error::VortexResult; +use crate::BitPacked; use crate::BitPackedArray; -impl ValidityHelper for BitPackedArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for BitPacked { + fn validity(array: &BitPackedArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index 94862ff26e6..197dec6e852 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -11,7 +11,6 @@ use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; @@ -28,10 +27,10 @@ pub fn delta_compress( // Fill-forward null values so that transposed deltas at null positions remain // small. Without this, bitpacking may skip patches for null positions, and the // corrupted delta values propagate through the cumulative sum during decompression. - let filled = fill_forward_nulls(array.to_buffer::(), array.validity()); + let filled = fill_forward_nulls(array.to_buffer::(), &array.validity()); let (bases, deltas) = compress_primitive::(&filled); // TODO(robert): This can be avoided if we add TransposedBoolArray that performs index translation when necessary. - let validity = transpose_validity(array.validity(), ctx)?; + let validity = transpose_validity(&array.validity(), ctx)?; ( PrimitiveArray::new(bases, array.dtype().nullability().into()), PrimitiveArray::new(deltas, validity), diff --git a/encodings/fastlanes/src/delta/array/delta_decompress.rs b/encodings/fastlanes/src/delta/array/delta_decompress.rs index c678b7d9b87..9ac446a92fc 100644 --- a/encodings/fastlanes/src/delta/array/delta_decompress.rs +++ b/encodings/fastlanes/src/delta/array/delta_decompress.rs @@ -12,7 +12,6 @@ use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; @@ -30,7 +29,7 @@ pub fn delta_decompress( let start = array.offset(); let end = start + array.len(); - let validity = untranspose_validity(deltas.validity(), ctx)?; + let validity = untranspose_validity(&deltas.validity(), ctx)?; let validity = validity.slice(start..end)?; Ok(match_each_unsigned_integer_ptype!(deltas.ptype(), |T| { diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index bffca15840b..d7633481b74 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -12,7 +12,6 @@ use vortex_array::dtype::PhysicalPType; use vortex_array::dtype::UnsignedPType; use vortex_array::match_each_integer_ptype; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -58,7 +57,7 @@ pub fn decompress(array: &FoRArray, ctx: &mut ExecutionCtx) -> VortexResult(ctx)?; - let validity = encoded.validity().clone(); + let validity = encoded.validity(); Ok(match_each_integer_ptype!(ptype, |T| { let min = array @@ -117,7 +116,7 @@ pub(crate) fn fused_decompress< // Decode all chunks (initial, full, and trailer) in one call. unpacked.decode_into(uninit_slice); - if let Some(patches) = bp.patches() { + if let Some(ref patches) = bp.patches() { bitpack_decompress::apply_patches_to_uninit_range_fn( &mut uninit_range, patches, diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index 51cf30c447e..9c0feb8291d 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -10,7 +10,6 @@ use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::NativePType; use vortex_array::match_each_native_ptype; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::BitBufferMut; use vortex_buffer::BufferMut; use vortex_error::VortexResult; @@ -36,7 +35,7 @@ where { // Fill-forward null values so the RLE encoder doesn't see garbage at null positions, // which would create spurious run boundaries and inflate the dictionary. - let values = fill_forward_nulls(array.to_buffer::(), array.validity()); + let values = fill_forward_nulls(array.to_buffer::(), &array.validity()); let len = values.len(); let padded_len = len.next_multiple_of(FL_CHUNK_SIZE); @@ -258,7 +257,7 @@ mod tests { let primitive = values.clone().into_array().to_primitive(); let result = RLEArray::encode(&primitive).unwrap(); let decoded = result.to_primitive(); - let expected = PrimitiveArray::new(values, primitive.validity().clone()); + let expected = PrimitiveArray::new(values, primitive.validity()); assert_arrays_eq!(decoded, expected); } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 72c8f9eebcd..448b45718da 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -39,7 +39,6 @@ use vortex_array::vtable::Array; use vortex_array::vtable::ArrayId; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; -use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::validity_to_child; use vortex_buffer::Buffer; @@ -453,7 +452,7 @@ impl FSSTArray { as Box Compressor + Send>)); let codes_array = codes.clone().into_array(); let codes_offsets_slot = Some(codes.offsets().clone()); - let codes_validity_slot = validity_to_child(codes.validity(), codes.len()); + let codes_validity_slot = validity_to_child(&codes.validity(), codes.len()); Self { dtype, diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index aabd0e14027..133628e82e3 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -12,7 +12,6 @@ use vortex_array::arrays::varbinview::build_views::BinaryView; use vortex_array::arrays::varbinview::build_views::MAX_BUFFER_LEN; use vortex_array::arrays::varbinview::build_views::build_views; use vortex_array::match_each_integer_ptype; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; @@ -32,7 +31,7 @@ pub(super) fn canonicalize_fsst( views, Arc::from(buffers), array.dtype().clone(), - array.codes().validity().clone(), + array.codes().validity(), ) .into_array() }) diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index c438a9ab01e..603582d7b78 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -72,7 +72,7 @@ impl LikeKernel for FSST { // directly without cloning the entire FSSTArray into an ArrayRef. let validity = array .codes() - .validity()? + .validity() .union_nullability(pattern_scalar.dtype().nullability()); Ok(Some(BoolArray::new(result, validity).into_array())) diff --git a/encodings/parquet-variant/src/validity.rs b/encodings/parquet-variant/src/validity.rs index 5257c7a9c78..7d163663517 100644 --- a/encodings/parquet-variant/src/validity.rs +++ b/encodings/parquet-variant/src/validity.rs @@ -7,7 +7,7 @@ use vortex_array::vtable::ValidityHelper; use crate::array::ParquetVariantArray; impl ValidityHelper for ParquetVariantArray { - fn validity(&self) -> &Validity { - &self.validity + fn validity(&self) -> Validity { + self.validity.clone() } } diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 9d65988d7c7..279ea465fe5 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -44,7 +44,6 @@ use vortex_array::vtable::Array; use vortex_array::vtable::ArrayId; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValiditySliceHelper; use vortex_array::vtable::ValidityVTableFromValiditySliceHelper; use vortex_array::vtable::validity_to_child; @@ -428,7 +427,7 @@ impl PcoArray { parray.dtype().clone(), metadata, parray.len(), - parray.validity().clone(), + parray.validity(), )) } diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index a73bbc7948f..7656a82397d 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -22,7 +22,6 @@ use vortex_array::serde::SerializeOptions; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index 0a841c28194..bdba123e3d7 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -18,7 +18,6 @@ use vortex_array::match_each_native_ptype; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; diff --git a/encodings/runend/src/compute/take.rs b/encodings/runend/src/compute/take.rs index e8ad6dfdf3c..f07a44cd3af 100644 --- a/encodings/runend/src/compute/take.rs +++ b/encodings/runend/src/compute/take.rs @@ -15,7 +15,6 @@ use vortex_array::search_sorted::SearchResult; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -50,7 +49,7 @@ impl TakeExecute for RunEnd { .collect::>>()? }); - take_indices_unchecked(array, &checked_indices, primitive_indices.validity()).map(Some) + take_indices_unchecked(array, &checked_indices, &primitive_indices.validity()).map(Some) } } diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index 488c20f8023..2fc6101fc7e 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -6,7 +6,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -16,7 +15,7 @@ use zigzag::ZigZag as ExternalZigZag; use crate::ZigZagArray; pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { - let validity = parray.validity().clone(); + let validity = parray.validity(); let encoded = match parray.ptype() { PType::I8 => zigzag_encode_primitive::(parray.into_buffer_mut(), validity), PType::I16 => zigzag_encode_primitive::(parray.into_buffer_mut(), validity), @@ -44,7 +43,7 @@ where } pub fn zigzag_decode(parray: PrimitiveArray) -> PrimitiveArray { - let validity = parray.validity().clone(); + let validity = parray.validity(); match parray.ptype() { PType::U8 => zigzag_decode_primitive::(parray.into_buffer_mut(), validity), PType::U16 => zigzag_decode_primitive::(parray.into_buffer_mut(), validity), diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 901034f2082..9888eabfe91 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -38,7 +38,6 @@ use vortex_array::vtable::Array; use vortex_array::vtable::ArrayId; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValiditySliceHelper; use vortex_array::vtable::ValidityVTableFromValiditySliceHelper; use vortex_array::vtable::validity_to_child; @@ -604,7 +603,7 @@ impl ZstdArray { dtype, metadata, parray.len(), - parray.validity().clone(), + parray.validity(), )) } @@ -695,7 +694,7 @@ impl ZstdArray { dtype, metadata, vbv.len(), - vbv.validity().clone(), + vbv.validity(), )) } diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index d17fe512824..802ef768d51 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -14,7 +14,6 @@ use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Alignment; use vortex_buffer::Buffer; use vortex_mask::Mask; @@ -86,7 +85,7 @@ fn test_zstd_with_validity_and_multi_frame() { assert!( decompressed .validity() - .mask_eq(array.validity(), &mut ctx) + .mask_eq(&array.validity(), &mut ctx) .unwrap() ); diff --git a/fuzz/src/array/fill_null.rs b/fuzz/src/array/fill_null.rs index 8dc89c8a682..5954d6982f7 100644 --- a/fuzz/src/array/fill_null.rs +++ b/fuzz/src/array/fill_null.rs @@ -17,7 +17,6 @@ use vortex_array::match_each_decimal_value_type; use vortex_array::match_each_native_ptype; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; -use vortex_array::vtable::ValidityHelper; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; @@ -176,7 +175,7 @@ fn fill_varbinview_array( result_nullability: Nullability, ) -> ArrayRef { match array.validity() { - Validity::NonNullable | Validity::AllValid => array.clone().into_array(), + Validity::NonNullable | Validity::AllValid => array.into_array(), Validity::AllInvalid => ConstantArray::new(fill_value.clone(), array.len()).into_array(), Validity::Array(validity_array) => { let validity_bool_array = validity_array.to_bool(); diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 170db177854..71049e8c748 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -60,11 +60,11 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); BoolArray::new(array.to_bit_buffer(), new_validity).into_array() } Canonical::Primitive(array) => { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); PrimitiveArray::from_buffer_handle( array.buffer_handle().clone(), array.ptype(), @@ -73,14 +73,14 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); match_each_decimal_value_type!(array.values_type(), |D| { DecimalArray::new(array.buffer::(), array.decimal_dtype(), new_validity) .into_array() }) } Canonical::VarBinView(array) => { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); VarBinViewArray::new_handle( array.views_handle().clone(), array.buffers().clone(), @@ -90,7 +90,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); // SAFETY: Since we are only masking the validity and everything else comes from an // already valid `ListViewArray`, all of the invariants are still upheld. @@ -106,7 +106,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); FixedSizeListArray::new( array.elements().clone(), array.list_size(), @@ -116,7 +116,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(array.validity(), mask); + let new_validity = mask_validity(&array.validity(), mask); StructArray::try_new_with_dtype( array.unmasked_fields().clone(), array.struct_fields().clone(), diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index edadd7edf8e..525953db085 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -898,7 +898,7 @@ pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -950,6 +950,10 @@ pub fn vortex_array::arrays::Bool::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut vortex_array::arrays::BoolArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Bool + +pub fn vortex_array::arrays::Bool::validity(array: &vortex_array::arrays::BoolArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::bool::BoolArray impl vortex_array::arrays::BoolArray @@ -980,6 +984,8 @@ pub fn vortex_array::arrays::BoolArray::try_new_from_handle(bits: vortex_array:: pub fn vortex_array::arrays::BoolArray::validate(bits: &vortex_buffer::bit::buf::BitBuffer, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::BoolArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::BoolArray pub fn vortex_array::arrays::BoolArray::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -1030,10 +1036,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::BoolArray pub fn vortex_array::arrays::BoolArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::BoolArray - -pub fn vortex_array::arrays::BoolArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::bool::BoolArrayParts pub vortex_array::arrays::bool::BoolArrayParts::bits: vortex_array::buffer::BufferHandle @@ -1510,7 +1512,7 @@ pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -1562,6 +1564,10 @@ pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut vortex_array::arrays::DecimalArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::validity(array: &vortex_array::arrays::DecimalArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::decimal::DecimalArray impl vortex_array::arrays::DecimalArray @@ -1598,6 +1604,8 @@ pub fn vortex_array::arrays::DecimalArray::try_new vortex_error::VortexResult +pub fn vortex_array::arrays::DecimalArray::validity(&self) -> vortex_array::validity::Validity + pub fn vortex_array::arrays::DecimalArray::values_type(&self) -> vortex_array::dtype::DecimalType impl vortex_array::arrays::DecimalArray @@ -1634,10 +1642,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::DecimalArray pub fn vortex_array::arrays::DecimalArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::DecimalArray - -pub fn vortex_array::arrays::DecimalArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::decimal::DecimalArrayParts pub vortex_array::arrays::decimal::DecimalArrayParts::decimal_dtype: vortex_array::dtype::DecimalDType @@ -2636,7 +2640,7 @@ pub fn vortex_array::arrays::FixedSizeListArray::into_array(self) -> vortex_arra impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::FixedSizeListArray -pub fn vortex_array::arrays::FixedSizeListArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::FixedSizeListArray::validity(&self) -> vortex_array::validity::Validity pub mod vortex_array::arrays::list @@ -2798,7 +2802,7 @@ pub fn vortex_array::arrays::ListArray::into_array(self) -> vortex_array::ArrayR impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::ListArray -pub fn vortex_array::arrays::ListArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::ListArray::validity(&self) -> vortex_array::validity::Validity pub struct vortex_array::arrays::list::ListArrayParts @@ -2988,7 +2992,7 @@ pub fn vortex_array::arrays::ListViewArray::into_array(self) -> vortex_array::Ar impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::ListViewArray -pub fn vortex_array::arrays::ListViewArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::ListViewArray::validity(&self) -> vortex_array::validity::Validity pub struct vortex_array::arrays::listview::ListViewArrayParts @@ -3052,7 +3056,7 @@ pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked -pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -3104,6 +3108,10 @@ pub fn vortex_array::arrays::Masked::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut vortex_array::arrays::MaskedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Masked + +pub fn vortex_array::arrays::Masked::validity(array: &vortex_array::arrays::MaskedArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::masked::MaskedArray impl vortex_array::arrays::MaskedArray @@ -3112,6 +3120,8 @@ pub fn vortex_array::arrays::MaskedArray::child(&self) -> &vortex_array::ArrayRe pub fn vortex_array::arrays::MaskedArray::try_new(child: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::MaskedArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::MaskedArray pub fn vortex_array::arrays::MaskedArray::to_array(&self) -> vortex_array::ArrayRef @@ -3142,10 +3152,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::MaskedArray pub fn vortex_array::arrays::MaskedArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::MaskedArray - -pub fn vortex_array::arrays::MaskedArray::validity(&self) -> &vortex_array::validity::Validity - pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub mod vortex_array::arrays::null @@ -3582,7 +3588,7 @@ pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive -pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -3634,6 +3640,10 @@ pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut vortex_array::arrays::PrimitiveArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::validity(array: &vortex_array::arrays::PrimitiveArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::primitive::PrimitiveArray impl vortex_array::arrays::PrimitiveArray @@ -3690,6 +3700,8 @@ impl vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::into_parts(self) -> vortex_array::arrays::primitive::PrimitiveArrayParts +pub fn vortex_array::arrays::PrimitiveArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3732,10 +3744,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::PrimitiveArray - -pub fn vortex_array::arrays::PrimitiveArray::validity(&self) -> &vortex_array::validity::Validity - impl core::iter::traits::collect::FromIterator for vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::from_iter>(iter: I) -> Self @@ -4386,7 +4394,7 @@ pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct -pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct pub fn vortex_array::arrays::Struct::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -4438,6 +4446,10 @@ pub fn vortex_array::arrays::Struct::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut vortex_array::arrays::StructArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Struct + +pub fn vortex_array::arrays::Struct::validity(array: &vortex_array::arrays::StructArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::struct_::StructArray impl vortex_array::arrays::StructArray @@ -4482,6 +4494,8 @@ pub fn vortex_array::arrays::StructArray::unmasked_fields(&self) -> alloc::sync: pub fn vortex_array::arrays::StructArray::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::StructArray::validity(&self) -> vortex_array::validity::Validity + pub fn vortex_array::arrays::StructArray::with_column(&self, name: impl core::convert::Into, array: vortex_array::ArrayRef) -> vortex_error::VortexResult impl vortex_array::arrays::StructArray @@ -4522,10 +4536,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::StructArray pub fn vortex_array::arrays::StructArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::StructArray - -pub fn vortex_array::arrays::StructArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::struct_::StructArrayParts pub vortex_array::arrays::struct_::StructArrayParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> @@ -4616,7 +4626,7 @@ pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -4668,6 +4678,10 @@ pub fn vortex_array::arrays::VarBin::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut vortex_array::arrays::VarBinArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::VarBin + +pub fn vortex_array::arrays::VarBin::validity(array: &vortex_array::arrays::VarBinArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::varbin::VarBinArray impl vortex_array::arrays::VarBinArray @@ -4706,6 +4720,8 @@ pub fn vortex_array::arrays::VarBinArray::try_new_from_handle(offsets: vortex_ar pub fn vortex_array::arrays::VarBinArray::validate(offsets: &vortex_array::ArrayRef, bytes: &vortex_array::buffer::BufferHandle, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::VarBinArray pub fn vortex_array::arrays::VarBinArray::to_array(&self) -> vortex_array::ArrayRef @@ -4784,10 +4800,6 @@ impl vortex_array::accessor::ArrayAccessor<[u8]> for vortex_array::arrays::VarBi pub fn vortex_array::arrays::VarBinArray::with_iterator(&self, f: F) -> R where F: for<'a> core::ops::function::FnOnce(&mut dyn core::iter::traits::iterator::Iterator>) -> R -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::VarBinArray - -pub fn vortex_array::arrays::VarBinArray::validity(&self) -> &vortex_array::validity::Validity - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::VarBinArray pub fn vortex_array::arrays::VarBinArray::from_iter>>(iter: T) -> Self @@ -5028,7 +5040,7 @@ pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadat pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView -pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -5080,6 +5092,10 @@ pub fn vortex_array::arrays::VarBinView::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut vortex_array::arrays::VarBinViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::validity(array: &vortex_array::arrays::VarBinViewArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::varbinview::VarBinViewArray impl vortex_array::arrays::VarBinViewArray @@ -5118,6 +5134,8 @@ pub fn vortex_array::arrays::VarBinViewArray::try_new_handle(views: vortex_array pub fn vortex_array::arrays::VarBinViewArray::validate(views: &vortex_buffer::buffer::Buffer, buffers: &alloc::sync::Arc<[vortex_buffer::ByteBuffer]>, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinViewArray::validity(&self) -> vortex_array::validity::Validity + pub fn vortex_array::arrays::VarBinViewArray::views(&self) -> &[vortex_array::arrays::varbinview::BinaryView] pub fn vortex_array::arrays::VarBinViewArray::views_handle(&self) -> &vortex_array::buffer::BufferHandle @@ -5178,10 +5196,6 @@ impl vortex_array::accessor::ArrayAccessor<[u8]> for vortex_array::arrays::VarBi pub fn vortex_array::arrays::VarBinViewArray::with_iterator core::ops::function::FnOnce(&mut dyn core::iter::traits::iterator::Iterator>) -> R, R>(&self, f: F) -> R -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::VarBinViewArray - -pub fn vortex_array::arrays::VarBinViewArray::validity(&self) -> &vortex_array::validity::Validity - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::VarBinViewArray pub fn vortex_array::arrays::VarBinViewArray::from_iter>>(iter: T) -> Self @@ -5378,7 +5392,7 @@ pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -5430,6 +5444,10 @@ pub fn vortex_array::arrays::Bool::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut vortex_array::arrays::BoolArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Bool + +pub fn vortex_array::arrays::Bool::validity(array: &vortex_array::arrays::BoolArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::BoolArray impl vortex_array::arrays::BoolArray @@ -5460,6 +5478,8 @@ pub fn vortex_array::arrays::BoolArray::try_new_from_handle(bits: vortex_array:: pub fn vortex_array::arrays::BoolArray::validate(bits: &vortex_buffer::bit::buf::BitBuffer, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::BoolArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::BoolArray pub fn vortex_array::arrays::BoolArray::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -5510,10 +5530,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::BoolArray pub fn vortex_array::arrays::BoolArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::BoolArray - -pub fn vortex_array::arrays::BoolArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::Chunked impl vortex_array::arrays::Chunked @@ -5896,7 +5912,7 @@ pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -5948,6 +5964,10 @@ pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut vortex_array::arrays::DecimalArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::validity(array: &vortex_array::arrays::DecimalArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::DecimalArray impl vortex_array::arrays::DecimalArray @@ -5984,6 +6004,8 @@ pub fn vortex_array::arrays::DecimalArray::try_new vortex_error::VortexResult +pub fn vortex_array::arrays::DecimalArray::validity(&self) -> vortex_array::validity::Validity + pub fn vortex_array::arrays::DecimalArray::values_type(&self) -> vortex_array::dtype::DecimalType impl vortex_array::arrays::DecimalArray @@ -6020,10 +6042,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::DecimalArray pub fn vortex_array::arrays::DecimalArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::DecimalArray - -pub fn vortex_array::arrays::DecimalArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::Dict impl vortex_array::arrays::dict::Dict @@ -6640,7 +6658,7 @@ pub fn vortex_array::arrays::FixedSizeListArray::into_array(self) -> vortex_arra impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::FixedSizeListArray -pub fn vortex_array::arrays::FixedSizeListArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::FixedSizeListArray::validity(&self) -> vortex_array::validity::Validity pub struct vortex_array::arrays::List @@ -6800,7 +6818,7 @@ pub fn vortex_array::arrays::ListArray::into_array(self) -> vortex_array::ArrayR impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::ListArray -pub fn vortex_array::arrays::ListArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::ListArray::validity(&self) -> vortex_array::validity::Validity pub struct vortex_array::arrays::ListView @@ -6968,7 +6986,7 @@ pub fn vortex_array::arrays::ListViewArray::into_array(self) -> vortex_array::Ar impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::ListViewArray -pub fn vortex_array::arrays::ListViewArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::ListViewArray::validity(&self) -> vortex_array::validity::Validity pub struct vortex_array::arrays::Masked @@ -7012,7 +7030,7 @@ pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked -pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -7064,6 +7082,10 @@ pub fn vortex_array::arrays::Masked::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut vortex_array::arrays::MaskedArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Masked + +pub fn vortex_array::arrays::Masked::validity(array: &vortex_array::arrays::MaskedArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::MaskedArray impl vortex_array::arrays::MaskedArray @@ -7072,6 +7094,8 @@ pub fn vortex_array::arrays::MaskedArray::child(&self) -> &vortex_array::ArrayRe pub fn vortex_array::arrays::MaskedArray::try_new(child: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::MaskedArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::MaskedArray pub fn vortex_array::arrays::MaskedArray::to_array(&self) -> vortex_array::ArrayRef @@ -7102,10 +7126,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::MaskedArray pub fn vortex_array::arrays::MaskedArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::MaskedArray - -pub fn vortex_array::arrays::MaskedArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::Null impl vortex_array::arrays::null::Null @@ -7448,7 +7468,7 @@ pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive -pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -7500,6 +7520,10 @@ pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut vortex_array::arrays::PrimitiveArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::validity(array: &vortex_array::arrays::PrimitiveArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::PrimitiveArray impl vortex_array::arrays::PrimitiveArray @@ -7556,6 +7580,8 @@ impl vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::into_parts(self) -> vortex_array::arrays::primitive::PrimitiveArrayParts +pub fn vortex_array::arrays::PrimitiveArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -7598,10 +7624,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::PrimitiveArray - -pub fn vortex_array::arrays::PrimitiveArray::validity(&self) -> &vortex_array::validity::Validity - impl core::iter::traits::collect::FromIterator for vortex_array::arrays::PrimitiveArray pub fn vortex_array::arrays::PrimitiveArray::from_iter>(iter: I) -> Self @@ -8044,7 +8066,7 @@ pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct -pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct pub fn vortex_array::arrays::Struct::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -8096,6 +8118,10 @@ pub fn vortex_array::arrays::Struct::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut vortex_array::arrays::StructArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Struct + +pub fn vortex_array::arrays::Struct::validity(array: &vortex_array::arrays::StructArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::StructArray impl vortex_array::arrays::StructArray @@ -8140,6 +8166,8 @@ pub fn vortex_array::arrays::StructArray::unmasked_fields(&self) -> alloc::sync: pub fn vortex_array::arrays::StructArray::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::StructArray::validity(&self) -> vortex_array::validity::Validity + pub fn vortex_array::arrays::StructArray::with_column(&self, name: impl core::convert::Into, array: vortex_array::ArrayRef) -> vortex_error::VortexResult impl vortex_array::arrays::StructArray @@ -8180,10 +8208,6 @@ impl vortex_array::IntoArray for vortex_array::arrays::StructArray pub fn vortex_array::arrays::StructArray::into_array(self) -> vortex_array::ArrayRef -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::StructArray - -pub fn vortex_array::arrays::StructArray::validity(&self) -> &vortex_array::validity::Validity - pub struct vortex_array::arrays::TemporalArray impl vortex_array::arrays::datetime::TemporalArray @@ -8298,7 +8322,7 @@ pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -8350,6 +8374,10 @@ pub fn vortex_array::arrays::VarBin::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut vortex_array::arrays::VarBinArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::VarBin + +pub fn vortex_array::arrays::VarBin::validity(array: &vortex_array::arrays::VarBinArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::VarBinArray impl vortex_array::arrays::VarBinArray @@ -8388,6 +8416,8 @@ pub fn vortex_array::arrays::VarBinArray::try_new_from_handle(offsets: vortex_ar pub fn vortex_array::arrays::VarBinArray::validate(offsets: &vortex_array::ArrayRef, bytes: &vortex_array::buffer::BufferHandle, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinArray::validity(&self) -> vortex_array::validity::Validity + impl vortex_array::arrays::VarBinArray pub fn vortex_array::arrays::VarBinArray::to_array(&self) -> vortex_array::ArrayRef @@ -8466,10 +8496,6 @@ impl vortex_array::accessor::ArrayAccessor<[u8]> for vortex_array::arrays::VarBi pub fn vortex_array::arrays::VarBinArray::with_iterator(&self, f: F) -> R where F: for<'a> core::ops::function::FnOnce(&mut dyn core::iter::traits::iterator::Iterator>) -> R -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::VarBinArray - -pub fn vortex_array::arrays::VarBinArray::validity(&self) -> &vortex_array::validity::Validity - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::VarBinArray pub fn vortex_array::arrays::VarBinArray::from_iter>>(iter: T) -> Self @@ -8524,7 +8550,7 @@ pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadat pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView -pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -8576,6 +8602,10 @@ pub fn vortex_array::arrays::VarBinView::vtable(_array: &Self::Array) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut vortex_array::arrays::VarBinViewArray, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::validity(array: &vortex_array::arrays::VarBinViewArray) -> vortex_error::VortexResult + pub struct vortex_array::arrays::VarBinViewArray impl vortex_array::arrays::VarBinViewArray @@ -8614,6 +8644,8 @@ pub fn vortex_array::arrays::VarBinViewArray::try_new_handle(views: vortex_array pub fn vortex_array::arrays::VarBinViewArray::validate(views: &vortex_buffer::buffer::Buffer, buffers: &alloc::sync::Arc<[vortex_buffer::ByteBuffer]>, dtype: &vortex_array::dtype::DType, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinViewArray::validity(&self) -> vortex_array::validity::Validity + pub fn vortex_array::arrays::VarBinViewArray::views(&self) -> &[vortex_array::arrays::varbinview::BinaryView] pub fn vortex_array::arrays::VarBinViewArray::views_handle(&self) -> &vortex_array::buffer::BufferHandle @@ -8674,10 +8706,6 @@ impl vortex_array::accessor::ArrayAccessor<[u8]> for vortex_array::arrays::VarBi pub fn vortex_array::arrays::VarBinViewArray::with_iterator core::ops::function::FnOnce(&mut dyn core::iter::traits::iterator::Iterator>) -> R, R>(&self, f: F) -> R -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::VarBinViewArray - -pub fn vortex_array::arrays::VarBinViewArray::validity(&self) -> &vortex_array::validity::Validity - impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::VarBinViewArray pub fn vortex_array::arrays::VarBinViewArray::from_iter>>(iter: T) -> Self @@ -21474,7 +21502,7 @@ pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -21654,7 +21682,7 @@ pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -22014,7 +22042,7 @@ pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked -pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -22074,7 +22102,7 @@ pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive -pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -22194,7 +22222,7 @@ pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct -pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct pub fn vortex_array::arrays::Struct::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -22254,7 +22282,7 @@ pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -22314,7 +22342,7 @@ pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadat pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView -pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -22918,7 +22946,7 @@ pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -23098,7 +23126,7 @@ pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -23458,7 +23486,7 @@ pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked -pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -23518,7 +23546,7 @@ pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive -pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -23638,7 +23666,7 @@ pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct -pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct pub fn vortex_array::arrays::Struct::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -23698,7 +23726,7 @@ pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata vortex_error::VortexResult<()> @@ -23758,7 +23786,7 @@ pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadat pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView -pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::vtable::ValidityVTableFromValidityHelper +pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::append_to_builder(array: &Self::Array, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -24190,47 +24218,19 @@ pub fn vortex_array::vtable::ValidityChildSliceHelper::unsliced_child_and_slice( pub trait vortex_array::vtable::ValidityHelper -pub fn vortex_array::vtable::ValidityHelper::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::BoolArray - -pub fn vortex_array::arrays::BoolArray::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::DecimalArray - -pub fn vortex_array::arrays::DecimalArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::vtable::ValidityHelper::validity(&self) -> vortex_array::validity::Validity impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::FixedSizeListArray -pub fn vortex_array::arrays::FixedSizeListArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::FixedSizeListArray::validity(&self) -> vortex_array::validity::Validity impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::ListArray -pub fn vortex_array::arrays::ListArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::ListArray::validity(&self) -> vortex_array::validity::Validity impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::ListViewArray -pub fn vortex_array::arrays::ListViewArray::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::MaskedArray - -pub fn vortex_array::arrays::MaskedArray::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::PrimitiveArray - -pub fn vortex_array::arrays::PrimitiveArray::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::StructArray - -pub fn vortex_array::arrays::StructArray::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::VarBinArray - -pub fn vortex_array::arrays::VarBinArray::validity(&self) -> &vortex_array::validity::Validity - -impl vortex_array::vtable::ValidityHelper for vortex_array::arrays::VarBinViewArray - -pub fn vortex_array::arrays::VarBinViewArray::validity(&self) -> &vortex_array::validity::Validity +pub fn vortex_array::arrays::ListViewArray::validity(&self) -> vortex_array::validity::Validity pub trait vortex_array::vtable::ValiditySliceHelper @@ -24242,6 +24242,10 @@ pub trait vortex_array::vtable::ValidityVTable pub fn vortex_array::vtable::ValidityVTable::validity(array: &::Array) -> vortex_error::VortexResult +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Bool + +pub fn vortex_array::arrays::Bool::validity(array: &vortex_array::arrays::BoolArray) -> vortex_error::VortexResult + impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Chunked pub fn vortex_array::arrays::Chunked::validity(array: &vortex_array::arrays::ChunkedArray) -> vortex_error::VortexResult @@ -24250,14 +24254,38 @@ impl vortex_array::vtable::ValidityVTable for vo pub fn vortex_array::arrays::Constant::validity(array: &vortex_array::arrays::ConstantArray) -> vortex_error::VortexResult +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Decimal + +pub fn vortex_array::arrays::Decimal::validity(array: &vortex_array::arrays::DecimalArray) -> vortex_error::VortexResult + impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Filter pub fn vortex_array::arrays::Filter::validity(array: &vortex_array::arrays::FilterArray) -> vortex_error::VortexResult +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Masked + +pub fn vortex_array::arrays::Masked::validity(array: &vortex_array::arrays::MaskedArray) -> vortex_error::VortexResult + +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Primitive + +pub fn vortex_array::arrays::Primitive::validity(array: &vortex_array::arrays::PrimitiveArray) -> vortex_error::VortexResult + impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Shared pub fn vortex_array::arrays::Shared::validity(array: &vortex_array::arrays::SharedArray) -> vortex_error::VortexResult +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Struct + +pub fn vortex_array::arrays::Struct::validity(array: &vortex_array::arrays::StructArray) -> vortex_error::VortexResult + +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::VarBin + +pub fn vortex_array::arrays::VarBin::validity(array: &vortex_array::arrays::VarBinArray) -> vortex_error::VortexResult + +impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::VarBinView + +pub fn vortex_array::arrays::VarBinView::validity(array: &vortex_array::arrays::VarBinViewArray) -> vortex_error::VortexResult + impl vortex_array::vtable::ValidityVTable for vortex_array::arrays::Variant pub fn vortex_array::arrays::Variant::validity(array: &::Array) -> vortex_error::VortexResult @@ -24294,6 +24322,8 @@ impl vortex_array::vtable::ValidityVTable for vortex_array::vtable::Validi pub fn vortex_array::vtable::ValidityVTableFromChild::validity(array: &::Array) -> vortex_error::VortexResult +pub fn vortex_array::vtable::child_to_validity(child: &core::option::Option, nullability: vortex_array::dtype::Nullability) -> vortex_array::validity::Validity + pub fn vortex_array::vtable::patches_child(patches: &vortex_array::patches::Patches, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::patches_child_name(idx: usize) -> &'static str diff --git a/vortex-array/src/aggregate_fn/accumulator_grouped.rs b/vortex-array/src/aggregate_fn/accumulator_grouped.rs index a4d9c38b60e..83b945708b2 100644 --- a/vortex-array/src/aggregate_fn/accumulator_grouped.rs +++ b/vortex-array/src/aggregate_fn/accumulator_grouped.rs @@ -180,7 +180,7 @@ impl GroupedAccumulator { elements.clone(), groups.offsets().clone(), groups.sizes().clone(), - groups.validity().clone(), + groups.validity(), ) }; kernel @@ -270,7 +270,7 @@ impl GroupedAccumulator { FixedSizeListArray::new_unchecked( elements.clone(), groups.list_size(), - groups.validity().clone(), + groups.validity(), groups.len(), ) }; diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index da65ab3e101..bc3d1830056 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -15,6 +15,7 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; pub(super) const VALIDITY_SLOT: usize = 0; @@ -60,7 +61,6 @@ pub struct BoolArray { pub(super) bits: BufferHandle, pub(super) offset: usize, pub(super) len: usize, - pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -115,7 +115,6 @@ impl BoolArray { bits: BufferHandle::new_host(buffer), offset, len, - validity, stats_set: ArrayStats::default(), }) } @@ -153,7 +152,6 @@ impl BoolArray { bits, offset, len, - validity, stats_set: ArrayStats::default(), }) } @@ -175,7 +173,6 @@ impl BoolArray { bits: BufferHandle::new_host(buffer), offset, len, - validity, stats_set: ArrayStats::default(), } } @@ -203,14 +200,20 @@ impl BoolArray { Ok(()) } + /// Reconstructs the validity from the slot state. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + /// Splits into owned parts #[inline] pub fn into_parts(self) -> BoolArrayParts { + let validity = self.validity(); BoolArrayParts { bits: self.bits, offset: self.offset, len: self.len, - validity: self.validity, + validity, } } @@ -330,7 +333,6 @@ mod tests { use crate::assert_arrays_eq; use crate::patches::Patches; use crate::validity::Validity; - use crate::vtable::ValidityHelper; #[test] fn bool_array() { diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs index 105625f3258..6f1f575aaeb 100644 --- a/vortex-array/src/arrays/bool/compute/cast.rs +++ b/vortex-array/src/arrays/bool/compute/cast.rs @@ -9,7 +9,6 @@ use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; -use crate::vtable::ValidityHelper; impl CastReduce for Bool { fn cast(array: &BoolArray, dtype: &DType) -> VortexResult> { @@ -20,7 +19,6 @@ impl CastReduce for Bool { let new_nullability = dtype.nullability(); let new_validity = array .validity() - .clone() .cast_nullability(new_nullability, array.len())?; Ok(Some( BoolArray::new(array.to_bit_buffer(), new_validity).into_array(), diff --git a/vortex-array/src/arrays/bool/compute/fill_null.rs b/vortex-array/src/arrays/bool/compute/fill_null.rs index b209cb5aa5d..d3ea9231e51 100644 --- a/vortex-array/src/arrays/bool/compute/fill_null.rs +++ b/vortex-array/src/arrays/bool/compute/fill_null.rs @@ -12,7 +12,6 @@ use crate::arrays::BoolArray; use crate::scalar::Scalar; use crate::scalar_fn::fns::fill_null::FillNullKernel; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl FillNullKernel for Bool { fn fill_null( diff --git a/vortex-array/src/arrays/bool/compute/filter.rs b/vortex-array/src/arrays/bool/compute/filter.rs index cf0f90f66ca..82962013ec7 100644 --- a/vortex-array/src/arrays/bool/compute/filter.rs +++ b/vortex-array/src/arrays/bool/compute/filter.rs @@ -14,7 +14,6 @@ use crate::IntoArray; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::filter::FilterReduce; -use crate::vtable::ValidityHelper; /// If the filter density is above 80%, we use slices to filter the array instead of indices. const FILTER_SLICES_DENSITY_THRESHOLD: f64 = 0.8; diff --git a/vortex-array/src/arrays/bool/compute/mask.rs b/vortex-array/src/arrays/bool/compute/mask.rs index 4ca590094f4..2570045c981 100644 --- a/vortex-array/src/arrays/bool/compute/mask.rs +++ b/vortex-array/src/arrays/bool/compute/mask.rs @@ -9,17 +9,13 @@ use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for Bool { fn mask(array: &BoolArray, mask: &ArrayRef) -> VortexResult> { Ok(Some( BoolArray::new( array.to_bit_buffer(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .into_array(), )) diff --git a/vortex-array/src/arrays/bool/compute/rules.rs b/vortex-array/src/arrays/bool/compute/rules.rs index c4ed7496229..0a36a8981ae 100644 --- a/vortex-array/src/arrays/bool/compute/rules.rs +++ b/vortex-array/src/arrays/bool/compute/rules.rs @@ -15,7 +15,6 @@ use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; -use crate::vtable::ValidityHelper; pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&BoolMaskedValidityRule), @@ -50,7 +49,7 @@ impl ArrayParentReduceRule for BoolMaskedValidityRule { Ok(Some( BoolArray::new( array.to_bit_buffer(), - array.validity().clone().and(parent.validity().clone())?, + array.validity().and(parent.validity())?, ) .into_array(), )) diff --git a/vortex-array/src/arrays/bool/compute/slice.rs b/vortex-array/src/arrays/bool/compute/slice.rs index 31045004c93..b6d9a8469f2 100644 --- a/vortex-array/src/arrays/bool/compute/slice.rs +++ b/vortex-array/src/arrays/bool/compute/slice.rs @@ -10,7 +10,6 @@ use crate::IntoArray; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::arrays::slice::SliceReduce; -use crate::vtable::ValidityHelper; impl SliceReduce for Bool { fn slice(array: &Self::Array, range: Range) -> VortexResult> { diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index be4a25a9ad4..75233a1b4e3 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -20,7 +20,6 @@ use crate::builtins::ArrayBuiltins; use crate::executor::ExecutionCtx; use crate::match_each_integer_ptype; use crate::scalar::Scalar; -use crate::vtable::ValidityHelper; impl TakeExecute for Bool { fn take( diff --git a/vortex-array/src/arrays/bool/patch.rs b/vortex-array/src/arrays/bool/patch.rs index 3e787bb278e..4d16a0e77e2 100644 --- a/vortex-array/src/arrays/bool/patch.rs +++ b/vortex-array/src/arrays/bool/patch.rs @@ -10,7 +10,6 @@ use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::match_each_unsigned_integer_ptype; use crate::patches::Patches; -use crate::vtable::ValidityHelper; impl BoolArray { pub fn patch(self, patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult { @@ -19,13 +18,9 @@ impl BoolArray { let indices = patches.indices().clone().execute::(ctx)?; let values = patches.values().clone().execute::(ctx)?; - let patched_validity = self.validity().clone().patch( - len, - offset, - patches.indices(), - values.validity(), - ctx, - )?; + let patched_validity = + self.validity() + .patch(len, offset, patches.indices(), &values.validity(), ctx)?; let bit_buffer = self.into_bit_buffer(); let mut own_values = bit_buffer diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index b4e08938b6d..5fa24c02778 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -20,7 +20,6 @@ use crate::SerializeMetadata; use crate::arrays::BoolArray; use crate::arrays::bool::array::NUM_SLOTS; use crate::arrays::bool::array::SLOT_NAMES; -use crate::arrays::bool::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; @@ -28,7 +27,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::Array; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; mod canonical; mod kernel; mod operations; @@ -57,7 +55,7 @@ impl VTable for Bool { type Metadata = ProstMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &Bool @@ -82,7 +80,7 @@ impl VTable for Bool { fn array_hash(array: &BoolArray, state: &mut H, precision: Precision) { array.dtype.hash(state); array.to_bit_buffer().array_hash(state, precision); - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &BoolArray, other: &BoolArray, precision: Precision) -> bool { @@ -92,7 +90,7 @@ impl VTable for Bool { array .to_bit_buffer() .array_eq(&other.to_bit_buffer(), precision) - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: &BoolArray) -> usize { @@ -175,10 +173,6 @@ impl VTable for Bool { NUM_SLOTS, slots.len() ); - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype().nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/bool/vtable/validity.rs b/vortex-array/src/arrays/bool/vtable/validity.rs index f923e5f6910..c49ba1343dc 100644 --- a/vortex-array/src/arrays/bool/vtable/validity.rs +++ b/vortex-array/src/arrays/bool/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + +use crate::arrays::bool::vtable::Bool; use crate::arrays::bool::vtable::BoolArray; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for BoolArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for Bool { + fn validity(array: &BoolArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrays/datetime/test.rs b/vortex-array/src/arrays/datetime/test.rs index b77da6b622a..4e284a80ba2 100644 --- a/vortex-array/src/arrays/datetime/test.rs +++ b/vortex-array/src/arrays/datetime/test.rs @@ -22,7 +22,6 @@ use crate::extension::datetime::TimestampOptions; use crate::hash::ArrayEq; use crate::scalar::Scalar; use crate::validity::Validity; -use crate::vtable::ValidityHelper; macro_rules! test_temporal_roundtrip { ($prim:ty, $constructor:expr, $unit:expr) => {{ diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index 66263455c37..90beb610f69 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -27,7 +27,7 @@ use crate::match_each_integer_ptype; use crate::patches::Patches; use crate::stats::ArrayStats; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; pub(super) const VALIDITY_SLOT: usize = 0; @@ -97,7 +97,6 @@ pub struct DecimalArray { pub(super) dtype: DType, pub(super) values: BufferHandle, pub(super) values_type: DecimalType, - pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -239,7 +238,6 @@ impl DecimalArray { values, values_type, dtype: DType::Decimal(decimal_dtype, validity.nullability()), - validity, stats_set: Default::default(), } } @@ -291,14 +289,20 @@ impl DecimalArray { } } + /// Reconstructs the validity from the slot state. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + pub fn into_parts(self) -> DecimalArrayParts { + let validity = self.validity(); let decimal_dtype = self.dtype.into_decimal_opt().vortex_expect("cannot fail"); DecimalArrayParts { decimal_dtype, values: self.values, values_type: self.values_type, - validity: self.validity, + validity, } } @@ -389,11 +393,11 @@ impl DecimalArray { let patch_indices = patches.indices().clone().execute::(ctx)?; let patch_values = patches.values().clone().execute::(ctx)?; - let patched_validity = self.validity().clone().patch( + let patched_validity = self.validity().patch( self.len(), offset, &patch_indices.clone().into_array(), - patch_values.validity(), + &patch_values.validity(), ctx, )?; assert_eq!(self.decimal_dtype(), patch_values.decimal_dtype()); diff --git a/vortex-array/src/arrays/decimal/compute/between.rs b/vortex-array/src/arrays/decimal/compute/between.rs index d504cbb8822..6bb3db961b6 100644 --- a/vortex-array/src/arrays/decimal/compute/between.rs +++ b/vortex-array/src/arrays/decimal/compute/between.rs @@ -18,7 +18,6 @@ use crate::scalar::Scalar; use crate::scalar_fn::fns::between::BetweenKernel; use crate::scalar_fn::fns::between::BetweenOptions; use crate::scalar_fn::fns::between::StrictComparison; -use crate::vtable::ValidityHelper; impl BetweenKernel for Decimal { fn between( @@ -107,7 +106,7 @@ fn between_impl( let value = buffer[idx]; lower_op(lower, value) & upper_op(value, upper) }), - arr.validity().clone().union_nullability(nullability), + arr.validity().union_nullability(nullability), ) .into_array() } diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index cea97ac91ae..9f613898926 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -17,7 +17,6 @@ use crate::dtype::DecimalType; use crate::dtype::NativeDecimalType; use crate::match_each_decimal_value_type; use crate::scalar_fn::fns::cast::CastKernel; -use crate::vtable::ValidityHelper; impl CastKernel for Decimal { fn cast( @@ -62,7 +61,6 @@ impl CastKernel for Decimal { // Cast the validity to the new nullability let new_validity = array .validity() - .clone() .cast_nullability(*to_nullability, array.len())?; // If the target needs a wider physical type, upcast the values @@ -120,7 +118,7 @@ pub fn upcast_decimal_values( } let decimal_dtype = array.decimal_dtype(); - let validity = array.validity().clone(); + let validity = array.validity(); // Use match_each_decimal_value_type to dispatch based on source and target types match_each_decimal_value_type!(from_values_type, |F| { @@ -156,7 +154,6 @@ mod tests { use crate::dtype::DecimalType; use crate::dtype::Nullability; use crate::validity::Validity; - use crate::vtable::ValidityHelper; #[test] fn cast_decimal_to_nullable() { diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index 1ef920ac74f..89e001ed2d1 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -21,7 +21,6 @@ use crate::scalar::DecimalValue; use crate::scalar::Scalar; use crate::scalar_fn::fns::fill_null::FillNullKernel; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl FillNullKernel for Decimal { fn fill_null( diff --git a/vortex-array/src/arrays/decimal/compute/mask.rs b/vortex-array/src/arrays/decimal/compute/mask.rs index d6d2c65368e..be5d1854524 100644 --- a/vortex-array/src/arrays/decimal/compute/mask.rs +++ b/vortex-array/src/arrays/decimal/compute/mask.rs @@ -10,7 +10,6 @@ use crate::arrays::DecimalArray; use crate::match_each_decimal_value_type; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for Decimal { fn mask(array: &DecimalArray, mask: &ArrayRef) -> VortexResult> { @@ -22,10 +21,7 @@ impl MaskReduce for Decimal { DecimalArray::new_unchecked( array.buffer::(), array.decimal_dtype(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) } .into_array() diff --git a/vortex-array/src/arrays/decimal/compute/rules.rs b/vortex-array/src/arrays/decimal/compute/rules.rs index a7d7dab51e0..8f99937d9f6 100644 --- a/vortex-array/src/arrays/decimal/compute/rules.rs +++ b/vortex-array/src/arrays/decimal/compute/rules.rs @@ -17,7 +17,6 @@ use crate::match_each_decimal_value_type; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; -use crate::vtable::ValidityHelper; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DecimalMaskedValidityRule), @@ -50,7 +49,7 @@ impl ArrayParentReduceRule for DecimalMaskedValidityRule { DecimalArray::new_unchecked( array.buffer::(), array.decimal_dtype(), - array.validity().clone().and(parent.validity().clone())?, + array.validity().and(parent.validity())?, ) } .into_array() @@ -64,7 +63,7 @@ impl SliceReduce for Decimal { fn slice(array: &Self::Array, range: Range) -> VortexResult> { let result = match_each_decimal_value_type!(array.values_type(), |D| { let sliced = array.buffer::().slice(range.clone()); - let validity = array.validity().clone().slice(range)?; + let validity = array.validity().slice(range)?; // SAFETY: Slicing preserves all DecimalArray invariants unsafe { DecimalArray::new_unchecked(sliced, array.decimal_dtype(), validity) } .into_array() diff --git a/vortex-array/src/arrays/decimal/compute/take.rs b/vortex-array/src/arrays/decimal/compute/take.rs index fb6b0470e00..4718d8ebe4e 100644 --- a/vortex-array/src/arrays/decimal/compute/take.rs +++ b/vortex-array/src/arrays/decimal/compute/take.rs @@ -15,7 +15,6 @@ use crate::dtype::NativeDecimalType; use crate::executor::ExecutionCtx; use crate::match_each_decimal_value_type; use crate::match_each_integer_ptype; -use crate::vtable::ValidityHelper; impl TakeExecute for Decimal { fn take( diff --git a/vortex-array/src/arrays/decimal/utils.rs b/vortex-array/src/arrays/decimal/utils.rs index 40dd7d9cff6..da505387d7e 100644 --- a/vortex-array/src/arrays/decimal/utils.rs +++ b/vortex-array/src/arrays/decimal/utils.rs @@ -8,7 +8,6 @@ use vortex_error::VortexExpect; use crate::arrays::DecimalArray; use crate::dtype::DecimalType; use crate::dtype::i256; -use crate::vtable::ValidityHelper; macro_rules! try_downcast { ($array:expr, from: $src:ty, to: $($dst:ty),*) => {{ @@ -29,7 +28,7 @@ macro_rules! try_downcast { .map(|v| <$dst as BigCast>::from(v).vortex_expect("decimal conversion failure")) .collect(), $array.decimal_dtype(), - $array.validity().clone(), + $array.validity(), ); } )* diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index d7cf27c37c6..8125dd85ea9 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -20,7 +20,6 @@ use crate::SerializeMetadata; use crate::arrays::DecimalArray; use crate::arrays::decimal::array::NUM_SLOTS; use crate::arrays::decimal::array::SLOT_NAMES; -use crate::arrays::decimal::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::DecimalType; @@ -31,7 +30,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::Array; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; mod kernel; mod operations; mod validity; @@ -58,7 +56,7 @@ impl VTable for Decimal { type Metadata = ProstMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &Decimal @@ -92,14 +90,14 @@ impl VTable for Decimal { array.dtype.hash(state); array.values.array_hash(state, precision); std::mem::discriminant(&array.values_type).hash(state); - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &DecimalArray, other: &DecimalArray, precision: Precision) -> bool { array.dtype == other.dtype && array.values.array_eq(&other.values, precision) && array.values_type == other.values_type - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: &DecimalArray) -> usize { @@ -192,10 +190,6 @@ impl VTable for Decimal { NUM_SLOTS, slots.len() ); - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/decimal/vtable/validity.rs b/vortex-array/src/arrays/decimal/vtable/validity.rs index 96956ac6aa8..da05b4aec58 100644 --- a/vortex-array/src/arrays/decimal/vtable/validity.rs +++ b/vortex-array/src/arrays/decimal/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + +use crate::arrays::decimal::vtable::Decimal; use crate::arrays::decimal::vtable::DecimalArray; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for DecimalArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for Decimal { + fn validity(array: &DecimalArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrays/filter/execute/bool.rs b/vortex-array/src/arrays/filter/execute/bool.rs index 834a11b2cee..78ebfcbebe1 100644 --- a/vortex-array/src/arrays/filter/execute/bool.rs +++ b/vortex-array/src/arrays/filter/execute/bool.rs @@ -3,7 +3,6 @@ use std::sync::Arc; -use vortex_error::VortexExpect; use vortex_mask::MaskValues; use crate::arrays::BoolArray; @@ -11,7 +10,7 @@ use crate::arrays::filter::execute::bitbuffer; use crate::arrays::filter::execute::filter_validity; pub fn filter_bool(array: &BoolArray, mask: &Arc) -> BoolArray { - let validity = array.validity().vortex_expect("missing BoolArray validity"); + let validity = array.validity(); let filtered_validity = filter_validity(validity, mask); let bit_buffer = array.to_bit_buffer(); diff --git a/vortex-array/src/arrays/filter/execute/decimal.rs b/vortex-array/src/arrays/filter/execute/decimal.rs index b3f1b755792..93aecee68b1 100644 --- a/vortex-array/src/arrays/filter/execute/decimal.rs +++ b/vortex-array/src/arrays/filter/execute/decimal.rs @@ -9,10 +9,9 @@ use crate::arrays::DecimalArray; use crate::arrays::filter::execute::buffer; use crate::arrays::filter::execute::filter_validity; use crate::match_each_decimal_value_type; -use crate::vtable::ValidityHelper; pub fn filter_decimal(array: &DecimalArray, mask: &Arc) -> DecimalArray { - let filtered_validity = filter_validity(array.validity().clone(), mask); + let filtered_validity = filter_validity(array.validity(), mask); match_each_decimal_value_type!(array.values_type(), |T| { let filtered_buffer = buffer::filter_buffer(array.buffer::(), mask.as_ref()); diff --git a/vortex-array/src/arrays/filter/execute/fixed_size_list.rs b/vortex-array/src/arrays/filter/execute/fixed_size_list.rs index 5b65ffee011..aede1ae550b 100644 --- a/vortex-array/src/arrays/filter/execute/fixed_size_list.rs +++ b/vortex-array/src/arrays/filter/execute/fixed_size_list.rs @@ -25,7 +25,7 @@ pub fn filter_fixed_size_list( array: &FixedSizeListArray, selection_mask: &Arc, ) -> FixedSizeListArray { - let filtered_validity = filter_validity(array.validity().clone(), selection_mask); + let filtered_validity = filter_validity(array.validity(), selection_mask); let elements = array.elements(); let new_len = selection_mask.true_count(); diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index 9d0434d0bd7..9459b710a72 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -41,7 +41,7 @@ pub fn filter_listview(array: &ListViewArray, selection_mask: &Arc) let offsets = array.offsets(); let sizes = array.sizes(); - let new_validity = filter_validity(array.validity().clone(), selection_mask); + let new_validity = filter_validity(array.validity(), selection_mask); debug_assert!( new_validity .maybe_len() diff --git a/vortex-array/src/arrays/filter/execute/primitive.rs b/vortex-array/src/arrays/filter/execute/primitive.rs index 7a88afdc942..cc02e2fb616 100644 --- a/vortex-array/src/arrays/filter/execute/primitive.rs +++ b/vortex-array/src/arrays/filter/execute/primitive.rs @@ -3,7 +3,6 @@ use std::sync::Arc; -use vortex_error::VortexExpect; use vortex_mask::MaskValues; use crate::arrays::PrimitiveArray; @@ -12,9 +11,7 @@ use crate::arrays::filter::execute::filter_validity; use crate::match_each_native_ptype; pub fn filter_primitive(array: &PrimitiveArray, mask: &Arc) -> PrimitiveArray { - let validity = array - .validity() - .vortex_expect("missing PrimitiveArray validity"); + let validity = array.validity(); let filtered_validity = filter_validity(validity, mask); match_each_native_ptype!(array.ptype(), |T| { diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index 15a2512f265..74540a8739d 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -10,10 +10,9 @@ use crate::ArrayRef; use crate::arrays::StructArray; use crate::arrays::filter::execute::filter_validity; use crate::arrays::filter::execute::values_to_mask; -use crate::vtable::ValidityHelper; pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray { - let filtered_validity = filter_validity(array.validity().clone(), mask); + let filtered_validity = filter_validity(array.validity(), mask); let mask_for_filter = values_to_mask(mask); let fields: Vec = array diff --git a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs index f990047dde2..1fce99ad918 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/cast.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/cast.rs @@ -25,7 +25,6 @@ impl CastReduce for FixedSizeList { let elements = array.elements().cast((**target_element_type).clone())?; let validity = array .validity() - .clone() .cast_nullability(dtype.nullability(), array.len())?; Ok(Some( diff --git a/vortex-array/src/arrays/fixed_size_list/compute/mask.rs b/vortex-array/src/arrays/fixed_size_list/compute/mask.rs index d189f5570be..bc4f44f75de 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/mask.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/mask.rs @@ -19,10 +19,7 @@ impl MaskReduce for FixedSizeList { FixedSizeListArray::new_unchecked( array.elements().clone(), array.list_size(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, array.len(), ) } diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs b/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs index 8a374872faf..75f84728eac 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/validity.rs @@ -6,7 +6,7 @@ use crate::validity::Validity; use crate::vtable::ValidityHelper; impl ValidityHelper for FixedSizeListArray { - fn validity(&self) -> &Validity { - &self.validity + fn validity(&self) -> Validity { + self.validity.clone() } } diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index 1577ac2b9eb..e0879c2e906 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -20,7 +20,6 @@ impl CastReduce for List { let validity = array .validity() - .clone() .cast_nullability(dtype.nullability(), array.len())?; let new_elements = array.elements().cast((**target_element_type).clone())?; diff --git a/vortex-array/src/arrays/list/compute/mask.rs b/vortex-array/src/arrays/list/compute/mask.rs index 46d5021033b..565540dd7db 100644 --- a/vortex-array/src/arrays/list/compute/mask.rs +++ b/vortex-array/src/arrays/list/compute/mask.rs @@ -16,10 +16,7 @@ impl MaskReduce for List { ListArray::try_new( array.elements().clone(), array.offsets().clone(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .map(|a| Some(a.into_array())) } diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index ada95d3154a..e6094bd4ef3 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -106,10 +106,7 @@ fn _take( Ok(ListArray::try_new( new_elements, new_offsets, - array - .validity() - .clone() - .take(&indices_array.clone().into_array())?, + array.validity().take(&indices_array.clone().into_array())?, )? .into_array()) } @@ -178,10 +175,7 @@ fn _take_nullable &Validity { - &self.validity + fn validity(&self) -> Validity { + self.validity.clone() } } diff --git a/vortex-array/src/arrays/listview/compute/cast.rs b/vortex-array/src/arrays/listview/compute/cast.rs index 06fb4f0bc8a..6b662827220 100644 --- a/vortex-array/src/arrays/listview/compute/cast.rs +++ b/vortex-array/src/arrays/listview/compute/cast.rs @@ -23,7 +23,6 @@ impl CastReduce for ListView { let new_elements = array.elements().cast((**target_element_type).clone())?; let validity = array .validity() - .clone() .cast_nullability(dtype.nullability(), array.len())?; // SAFETY: Since `cast` is length-preserving, all of the invariants remain the same. diff --git a/vortex-array/src/arrays/listview/compute/mask.rs b/vortex-array/src/arrays/listview/compute/mask.rs index 5b486f5a542..6732fc25f66 100644 --- a/vortex-array/src/arrays/listview/compute/mask.rs +++ b/vortex-array/src/arrays/listview/compute/mask.rs @@ -20,10 +20,7 @@ impl MaskReduce for ListView { array.elements().clone(), array.offsets().clone(), array.sizes().clone(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .with_zero_copy_to_list(array.is_zero_copy_to_list()) } diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index cddee9a2ec5..fb34d9b45e8 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -61,7 +61,7 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes list.elements().clone(), adjusted_offsets, sizes, - list.validity().clone(), + list.validity(), ) .with_zero_copy_to_list(true) }) @@ -124,7 +124,7 @@ pub fn list_from_list_view(list_view: ListViewArray) -> VortexResult ListArray::new_unchecked( zctl_array.elements().clone(), list_offsets, - zctl_array.validity().clone(), + zctl_array.validity(), ) }) } @@ -203,7 +203,7 @@ pub fn recursive_list_from_list_view(array: ArrayRef) -> VortexResult converted_elements, listview.offsets().clone(), listview.sizes().clone(), - listview.validity().clone(), + listview.validity(), ) .with_zero_copy_to_list(listview.is_zero_copy_to_list()) } @@ -224,7 +224,7 @@ pub fn recursive_list_from_list_view(array: ArrayRef) -> VortexResult FixedSizeListArray::try_new( converted_elements, fixed_size_list.list_size(), - fixed_size_list.validity().clone(), + fixed_size_list.validity(), fixed_size_list.len(), ) .vortex_expect( @@ -252,7 +252,7 @@ pub fn recursive_list_from_list_view(array: ArrayRef) -> VortexResult struct_array.names().clone(), converted_fields, struct_array.len(), - struct_array.validity().clone(), + struct_array.validity(), ) .vortex_expect("StructArray reconstruction should not fail with valid components") .into_array() diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index 0f094b40322..cf2725e5c68 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -351,7 +351,7 @@ impl ListViewArray { sliced_elements, adjusted_offsets, self.sizes().clone(), - self.validity().clone(), + self.validity(), ) .with_zero_copy_to_list(self.is_zero_copy_to_list()) }) diff --git a/vortex-array/src/arrays/listview/vtable/validity.rs b/vortex-array/src/arrays/listview/vtable/validity.rs index 4a7fe9db11a..b0ed6ec6a1a 100644 --- a/vortex-array/src/arrays/listview/vtable/validity.rs +++ b/vortex-array/src/arrays/listview/vtable/validity.rs @@ -6,7 +6,7 @@ use crate::validity::Validity; use crate::vtable::ValidityHelper; impl ValidityHelper for ListViewArray { - fn validity(&self) -> &Validity { - &self.validity + fn validity(&self) -> Validity { + self.validity.clone() } } diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 437be1fc8ce..0f2bae3af23 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -9,6 +9,7 @@ use crate::ArrayRef; use crate::dtype::DType; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; pub(super) const CHILD_SLOT: usize = 0; @@ -19,7 +20,6 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child", "validity"]; #[derive(Clone, Debug)] pub struct MaskedArray { pub(super) slots: Vec>, - pub(super) validity: Validity, pub(super) dtype: DType, pub(super) stats: ArrayStats, } @@ -48,12 +48,16 @@ impl MaskedArray { Ok(Self { slots: vec![Some(child), validity_slot], - validity, dtype, stats: ArrayStats::default(), }) } + /// Reconstructs the validity from the slots. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + pub fn child(&self) -> &ArrayRef { self.slots[CHILD_SLOT] .as_ref() diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index 66e6e0c02cb..8cf9b423f4f 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -9,7 +9,6 @@ use crate::IntoArray; use crate::arrays::Masked; use crate::arrays::MaskedArray; use crate::arrays::filter::FilterReduce; -use crate::vtable::ValidityHelper; impl FilterReduce for Masked { fn filter(array: &MaskedArray, mask: &Mask) -> VortexResult> { diff --git a/vortex-array/src/arrays/masked/compute/mask.rs b/vortex-array/src/arrays/masked/compute/mask.rs index 86d6e142afa..c23f499d71a 100644 --- a/vortex-array/src/arrays/masked/compute/mask.rs +++ b/vortex-array/src/arrays/masked/compute/mask.rs @@ -11,14 +11,12 @@ use crate::scalar_fn::EmptyOptions; use crate::scalar_fn::fns::mask::Mask as MaskExpr; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for Masked { fn mask(array: &MaskedArray, mask: &ArrayRef) -> VortexResult> { // AND the existing validity mask with the new mask and push into child. let combined_mask = array .validity() - .clone() .and(Validity::Array(mask.clone()))? .to_array(array.len()); let masked_child = MaskExpr.try_new_array( diff --git a/vortex-array/src/arrays/masked/compute/slice.rs b/vortex-array/src/arrays/masked/compute/slice.rs index 870cbec21f9..19044da35c1 100644 --- a/vortex-array/src/arrays/masked/compute/slice.rs +++ b/vortex-array/src/arrays/masked/compute/slice.rs @@ -14,7 +14,7 @@ use crate::arrays::slice::SliceReduce; impl SliceReduce for Masked { fn slice(array: &Self::Array, range: Range) -> VortexResult> { let child = array.child().slice(range.clone())?; - let validity = array.validity.slice(range)?; + let validity = array.validity().slice(range)?; Ok(Some(MaskedArray::try_new(child, validity)?.into_array())) } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index 565349c4fc5..98d4862e189 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -11,7 +11,6 @@ use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; -use crate::vtable::ValidityHelper; impl TakeReduce for Masked { fn take(array: &MaskedArray, indices: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/arrays/masked/execute.rs b/vortex-array/src/arrays/masked/execute.rs index a1b49fedef5..73c379cdc46 100644 --- a/vortex-array/src/arrays/masked/execute.rs +++ b/vortex-array/src/arrays/masked/execute.rs @@ -24,7 +24,6 @@ use crate::dtype::Nullability; use crate::executor::ExecutionCtx; use crate::match_each_decimal_value_type; use crate::validity::Validity; -use crate::vtable::ValidityHelper; /// TODO: replace usage of compute fn. /// Apply a validity mask to a canonical array, ANDing with existing validity. @@ -81,7 +80,7 @@ fn mask_validity_bool( ctx: &mut ExecutionCtx, ) -> VortexResult { let len = array.len(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; Ok(BoolArray::new(array.to_bit_buffer(), new_validity)) } @@ -92,7 +91,7 @@ fn mask_validity_primitive( ) -> VortexResult { let len = array.len(); let ptype = array.ptype(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; // SAFETY: validity has same length as values Ok(unsafe { PrimitiveArray::new_unchecked_from_handle( @@ -111,7 +110,7 @@ fn mask_validity_decimal( let len = array.len(); let dec_dtype = array.decimal_dtype(); let values_type = array.values_type(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; // SAFETY: We're only changing validity, not the data structure Ok(match_each_decimal_value_type!(values_type, |T| { let buffer = array.buffer::(); @@ -127,7 +126,7 @@ fn mask_validity_varbinview( ) -> VortexResult { let len = array.len(); let dtype = array.dtype().as_nullable(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; // SAFETY: We're only changing validity, not the data structure Ok(unsafe { VarBinViewArray::new_handle_unchecked( @@ -145,7 +144,7 @@ fn mask_validity_listview( ctx: &mut ExecutionCtx, ) -> VortexResult { let len = array.len(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity()?, mask, len, ctx)?; // SAFETY: We're only changing validity, not the data structure Ok(unsafe { ListViewArray::new_unchecked( @@ -164,7 +163,7 @@ fn mask_validity_fixed_size_list( ) -> VortexResult { let len = array.len(); let list_size = array.list_size(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity()?, mask, len, ctx)?; // SAFETY: We're only changing validity, not the data structure Ok(unsafe { FixedSizeListArray::new_unchecked(array.elements().clone(), list_size, new_validity, len) @@ -177,7 +176,7 @@ fn mask_validity_struct( ctx: &mut ExecutionCtx, ) -> VortexResult { let len = array.len(); - let new_validity = combine_validity(array.validity(), mask, len, ctx)?; + let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; let fields = array.unmasked_fields().clone(); let struct_fields = array.struct_fields().clone(); // SAFETY: We're only changing validity, not the data structure diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index 6e3452b3568..2046ce691f1 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -101,11 +101,5 @@ fn test_masked_child_preserves_length(#[case] validity: Validity) { assert_eq!(array.len(), len); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - assert!( - array - .validity() - .unwrap() - .mask_eq(&validity, &mut ctx) - .unwrap(), - ); + assert!(array.validity().mask_eq(&validity, &mut ctx).unwrap(),); } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index b326055ba02..3b8b3b792f8 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -22,7 +22,6 @@ use crate::arrays::ConstantArray; use crate::arrays::MaskedArray; use crate::arrays::masked::array::NUM_SLOTS; use crate::arrays::masked::array::SLOT_NAMES; -use crate::arrays::masked::array::VALIDITY_SLOT; use crate::arrays::masked::compute::rules::PARENT_RULES; use crate::arrays::masked::mask_validity_canonical; use crate::buffer::BufferHandle; @@ -39,7 +38,6 @@ use crate::vtable; use crate::vtable::Array; use crate::vtable::ArrayId; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; vtable!(Masked); #[derive(Clone, Debug)] @@ -54,7 +52,7 @@ impl VTable for Masked { type Metadata = EmptyMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &Masked @@ -78,13 +76,13 @@ impl VTable for Masked { fn array_hash(array: &MaskedArray, state: &mut H, precision: Precision) { array.child().array_hash(state, precision); - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); array.dtype.hash(state); } fn array_eq(array: &MaskedArray, other: &MaskedArray, precision: Precision) -> bool { array.child().array_eq(other.child(), precision) - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) && array.dtype == other.dtype } @@ -193,10 +191,6 @@ impl VTable for Masked { NUM_SLOTS, slots.len() ); - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/masked/vtable/validity.rs b/vortex-array/src/arrays/masked/vtable/validity.rs index 91b36534c66..5f020d6708a 100644 --- a/vortex-array/src/arrays/masked/vtable/validity.rs +++ b/vortex-array/src/arrays/masked/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + use crate::arrays::MaskedArray; +use crate::arrays::masked::vtable::Masked; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for MaskedArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for Masked { + fn validity(array: &MaskedArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrays/primitive/array/accessor.rs b/vortex-array/src/arrays/primitive/array/accessor.rs index a3b251e8b5f..26f284eb9e0 100644 --- a/vortex-array/src/arrays/primitive/array/accessor.rs +++ b/vortex-array/src/arrays/primitive/array/accessor.rs @@ -8,7 +8,6 @@ use crate::accessor::ArrayAccessor; use crate::arrays::PrimitiveArray; use crate::dtype::NativePType; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl ArrayAccessor for PrimitiveArray { fn with_iterator(&self, f: F) -> R diff --git a/vortex-array/src/arrays/primitive/array/cast.rs b/vortex-array/src/arrays/primitive/array/cast.rs index bb900863949..feb31aed8e9 100644 --- a/vortex-array/src/arrays/primitive/array/cast.rs +++ b/vortex-array/src/arrays/primitive/array/cast.rs @@ -16,7 +16,6 @@ use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::PType; -use crate::vtable::ValidityHelper; impl PrimitiveArray { /// Return a slice of the array's buffer. @@ -56,11 +55,7 @@ impl PrimitiveArray { "can't reinterpret cast between integers of two different widths" ); - PrimitiveArray::from_buffer_handle( - self.buffer_handle().clone(), - ptype, - self.validity().clone(), - ) + PrimitiveArray::from_buffer_handle(self.buffer_handle().clone(), ptype, self.validity()) } /// Narrow the array to the smallest possible integer type that can represent all values. @@ -73,7 +68,7 @@ impl PrimitiveArray { let Some(min_max) = min_max(&self.clone().into_array(), &mut ctx)? else { return Ok(PrimitiveArray::new( Buffer::::zeroed(self.len()), - self.validity.clone(), + self.validity(), )); }; @@ -174,7 +169,7 @@ mod tests { result.dtype(), &DType::Primitive(PType::U8, Nullability::Nullable) ); - assert!(matches!(result.validity, Validity::AllInvalid)); + assert!(matches!(result.validity(), Validity::AllInvalid)); } #[rstest] @@ -220,7 +215,7 @@ mod tests { &DType::Primitive(PType::U8, Nullability::Nullable) ); // Check that validity is preserved (the array should still have nullable values) - assert!(matches!(&result.validity, Validity::Array(_))); + assert!(matches!(&result.validity(), Validity::Array(_))); } #[test] @@ -257,7 +252,7 @@ mod tests { let array2 = PrimitiveArray::new(Buffer::::empty(), Validity::NonNullable); let result2 = array2.narrow().unwrap(); // Empty arrays should not have their validity changed - assert!(matches!(result.validity, Validity::AllInvalid)); - assert!(matches!(result2.validity, Validity::NonNullable)); + assert!(matches!(result.validity(), Validity::AllInvalid)); + assert!(matches!(result2.validity(), Validity::NonNullable)); } } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index ac9284e4856..78d48b52ad1 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -20,7 +20,6 @@ use crate::dtype::PType; use crate::match_each_native_ptype; use crate::stats::ArrayStats; use crate::validity::Validity; -use crate::vtable::ValidityHelper; mod accessor; mod cast; @@ -33,6 +32,7 @@ pub use patch::patch_chunk; use crate::ArrayRef; use crate::buffer::BufferHandle; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; pub(super) const VALIDITY_SLOT: usize = 0; @@ -79,7 +79,6 @@ pub struct PrimitiveArray { pub(super) slots: Vec>, pub(super) dtype: DType, pub(super) buffer: BufferHandle, - pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -111,7 +110,6 @@ impl PrimitiveArray { slots: Self::make_slots(&validity, len), buffer: handle, dtype: DType::Primitive(ptype, validity.nullability()), - validity, stats_set: ArrayStats::default(), } } @@ -166,7 +164,6 @@ impl PrimitiveArray { slots: Self::make_slots(&validity, len), dtype: DType::Primitive(T::PTYPE, validity.nullability()), buffer: BufferHandle::new_host(buffer.into_byte_buffer()), - validity, stats_set: Default::default(), } } @@ -195,13 +192,19 @@ impl PrimitiveArray { } impl PrimitiveArray { + /// Reconstructs the validity from the slot state. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + /// Consume the primitive array and returns its component parts. pub fn into_parts(self) -> PrimitiveArrayParts { let ptype = self.ptype(); + let validity = self.validity(); PrimitiveArrayParts { ptype, buffer: self.buffer, - validity: self.validity, + validity, } } } @@ -223,7 +226,6 @@ impl PrimitiveArray { slots: Self::make_slots(&validity, len), buffer: handle, dtype, - validity, stats_set: ArrayStats::default(), } } @@ -273,7 +275,7 @@ impl PrimitiveArray { R: NativePType, F: FnMut(T) -> R, { - let validity = self.validity().clone(); + let validity = self.validity(); let buffer = match self.try_into_buffer_mut() { Ok(buffer_mut) => buffer_mut.map_each_in_place(f), Err(buffer) => BufferMut::from_iter(buffer.iter().copied().map(f)), @@ -307,6 +309,6 @@ impl PrimitiveArray { BufferMut::::from_iter(buf_iter.zip(val.iter()).map(f)) } }; - Ok(PrimitiveArray::new(buffer.freeze(), validity.clone())) + Ok(PrimitiveArray::new(buffer.freeze(), validity)) } } diff --git a/vortex-array/src/arrays/primitive/array/patch.rs b/vortex-array/src/arrays/primitive/array/patch.rs index 29e04e63f9c..a8b1479ed80 100644 --- a/vortex-array/src/arrays/primitive/array/patch.rs +++ b/vortex-array/src/arrays/primitive/array/patch.rs @@ -16,18 +16,17 @@ use crate::match_each_native_ptype; use crate::patches::PATCH_CHUNK_SIZE; use crate::patches::Patches; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl PrimitiveArray { pub fn patch(self, patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult { let patch_indices = patches.indices().clone().execute::(ctx)?; let patch_values = patches.values().clone().execute::(ctx)?; - let patched_validity = self.validity().clone().patch( + let patched_validity = self.validity().patch( self.len(), patches.offset(), &patch_indices.clone().into_array(), - patch_values.validity(), + &patch_values.validity(), ctx, )?; Ok(match_each_integer_ptype!(patch_indices.ptype(), |I| { diff --git a/vortex-array/src/arrays/primitive/compute/between.rs b/vortex-array/src/arrays/primitive/compute/between.rs index a0ae640766d..29688af3056 100644 --- a/vortex-array/src/arrays/primitive/compute/between.rs +++ b/vortex-array/src/arrays/primitive/compute/between.rs @@ -16,7 +16,6 @@ use crate::match_each_native_ptype; use crate::scalar_fn::fns::between::BetweenKernel; use crate::scalar_fn::fns::between::BetweenOptions; use crate::scalar_fn::fns::between::StrictComparison; -use crate::vtable::ValidityHelper; impl BetweenKernel for Primitive { fn between( @@ -110,7 +109,7 @@ where let i = unsafe { *slice.get_unchecked(idx) }; lower_fn(lower, i) & upper_fn(i, upper) }), - arr.validity().clone().union_nullability(nullability), + arr.validity().union_nullability(nullability), ) .into_array() } diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index 17d6c20d7df..4893ba01f65 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -21,7 +21,6 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::match_each_native_ptype; use crate::scalar_fn::fns::cast::CastKernel; -use crate::vtable::ValidityHelper; impl CastKernel for Primitive { fn cast( @@ -37,7 +36,6 @@ impl CastKernel for Primitive { // First, check that the cast is compatible with the source array's validity let new_validity = array .validity() - .clone() .cast_nullability(new_nullability, array.len())?; // Same ptype: zero-copy, just update validity. @@ -144,7 +142,6 @@ mod test { use crate::dtype::Nullability; use crate::dtype::PType; use crate::validity::Validity; - use crate::vtable::ValidityHelper; #[allow(clippy::cognitive_complexity)] #[test] diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index 8889c82c483..088e88f8b07 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -16,7 +16,6 @@ use crate::match_each_native_ptype; use crate::scalar::Scalar; use crate::scalar_fn::fns::fill_null::FillNullKernel; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl FillNullKernel for Primitive { fn fill_null( diff --git a/vortex-array/src/arrays/primitive/compute/mask.rs b/vortex-array/src/arrays/primitive/compute/mask.rs index c8dcffe62ba..1fb20d696ba 100644 --- a/vortex-array/src/arrays/primitive/compute/mask.rs +++ b/vortex-array/src/arrays/primitive/compute/mask.rs @@ -9,7 +9,6 @@ use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for Primitive { fn mask(array: &PrimitiveArray, mask: &ArrayRef) -> VortexResult> { @@ -18,10 +17,7 @@ impl MaskReduce for Primitive { PrimitiveArray::new_unchecked_from_handle( array.buffer_handle().clone(), array.ptype(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .into_array() })) diff --git a/vortex-array/src/arrays/primitive/compute/rules.rs b/vortex-array/src/arrays/primitive/compute/rules.rs index df6eb35d888..42c0aea06fc 100644 --- a/vortex-array/src/arrays/primitive/compute/rules.rs +++ b/vortex-array/src/arrays/primitive/compute/rules.rs @@ -13,7 +13,6 @@ use crate::arrays::slice::SliceReduceAdaptor; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; -use crate::vtable::ValidityHelper; pub(crate) const RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&PrimitiveMaskedValidityRule), @@ -39,7 +38,7 @@ impl ArrayParentReduceRule for PrimitiveMaskedValidityRule { ) -> VortexResult> { // TODO(joe): make this lazy // Merge the parent's validity mask into the child's validity - let new_validity = array.validity().clone().and(parent.validity().clone())?; + let new_validity = array.validity().and(parent.validity())?; // SAFETY: masking validity does not change PrimitiveArray invariants let masked_array = unsafe { diff --git a/vortex-array/src/arrays/primitive/compute/slice.rs b/vortex-array/src/arrays/primitive/compute/slice.rs index 2844163e557..0db4308701a 100644 --- a/vortex-array/src/arrays/primitive/compute/slice.rs +++ b/vortex-array/src/arrays/primitive/compute/slice.rs @@ -12,7 +12,6 @@ use crate::arrays::PrimitiveArray; use crate::arrays::slice::SliceReduce; use crate::dtype::NativePType; use crate::match_each_native_ptype; -use crate::vtable::ValidityHelper; impl SliceReduce for Primitive { fn slice(array: &Self::Array, range: Range) -> VortexResult> { diff --git a/vortex-array/src/arrays/primitive/compute/take/mod.rs b/vortex-array/src/arrays/primitive/compute/take/mod.rs index 7eb0cc0161d..787278069c0 100644 --- a/vortex-array/src/arrays/primitive/compute/take/mod.rs +++ b/vortex-array/src/arrays/primitive/compute/take/mod.rs @@ -28,7 +28,6 @@ use crate::executor::ExecutionCtx; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; use crate::validity::Validity; -use crate::vtable::ValidityHelper; // Kernel selection happens on the first call to `take` and uses a combination of compile-time // and runtime feature detection to infer the best kernel for the platform. diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index a8d875a2345..0fed78614e2 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -16,7 +16,6 @@ use crate::ExecutionResult; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::array::NUM_SLOTS; use crate::arrays::primitive::array::SLOT_NAMES; -use crate::arrays::primitive::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::PType; @@ -25,7 +24,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::Array; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; mod kernel; mod operations; mod validity; @@ -50,7 +48,7 @@ impl VTable for Primitive { type Metadata = EmptyMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &Primitive @@ -75,13 +73,13 @@ impl VTable for Primitive { fn array_hash(array: &PrimitiveArray, state: &mut H, precision: Precision) { array.dtype.hash(state); array.buffer.array_hash(state, precision); - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &PrimitiveArray, other: &PrimitiveArray, precision: Precision) -> bool { array.dtype == other.dtype && array.buffer.array_eq(&other.buffer, precision) - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: &PrimitiveArray) -> usize { @@ -188,10 +186,6 @@ impl VTable for Primitive { NUM_SLOTS, slots.len() ); - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype().nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/primitive/vtable/validity.rs b/vortex-array/src/arrays/primitive/vtable/validity.rs index 8c1c3df5b27..bccfdab660e 100644 --- a/vortex-array/src/arrays/primitive/vtable/validity.rs +++ b/vortex-array/src/arrays/primitive/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::arrays::primitive::vtable::PrimitiveArray; +use vortex_error::VortexResult; + +use crate::arrays::PrimitiveArray; +use crate::arrays::primitive::vtable::Primitive; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for PrimitiveArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for Primitive { + fn validity(array: &PrimitiveArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 7756e3afe72..85f54604ea1 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -19,7 +19,7 @@ use crate::dtype::FieldNames; use crate::dtype::StructFields; use crate::stats::ArrayStats; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; // StructArray has a variable number of slots: [validity?, field_0, ..., field_N] @@ -149,7 +149,6 @@ pub struct StructArray { pub(super) len: usize, pub(super) dtype: DType, pub(super) slots: Vec>, - pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -213,6 +212,11 @@ impl StructArray { struct_dtype } + /// Reconstructs the validity from the slots. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + /// Create a new `StructArray` with the given length, but without any fields. pub fn new_fieldless_with_len(len: usize) -> Self { Self::try_new( @@ -310,7 +314,6 @@ impl StructArray { len: length, dtype: DType::Struct(dtype, validity.nullability()), slots, - validity, stats_set: Default::default(), } } @@ -382,6 +385,7 @@ impl StructArray { } pub fn into_parts(self) -> StructArrayParts { + let validity = self.validity(); let struct_fields = self.dtype.into_struct_fields(); let fields: Arc<[ArrayRef]> = self .slots @@ -392,7 +396,7 @@ impl StructArray { StructArrayParts { struct_fields, fields, - validity: self.validity, + validity, } } @@ -461,7 +465,7 @@ impl StructArray { FieldNames::from(names.as_slice()), children, self.len(), - self.validity().clone(), + self.validity(), ) } @@ -513,6 +517,6 @@ impl StructArray { .chain(once(array)) .collect(); - Self::try_new_with_dtype(children, new_fields, self.len, self.validity.clone()) + Self::try_new_with_dtype(children, new_fields, self.len, self.validity()) } } diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 272e6965164..a1c547cb865 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -15,7 +15,6 @@ use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar::Scalar; use crate::scalar_fn::fns::cast::CastKernel; -use crate::vtable::ValidityHelper; impl CastKernel for Struct { fn cast( @@ -72,7 +71,6 @@ impl CastKernel for Struct { let validity = array .validity() - .clone() .cast_nullability(dtype.nullability(), array.len())?; StructArray::try_new( diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index e8896da3cf7..09f43e6b981 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -9,7 +9,6 @@ use crate::arrays::Struct; use crate::arrays::StructArray; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for Struct { fn mask(array: &StructArray, mask: &ArrayRef) -> VortexResult> { @@ -17,10 +16,7 @@ impl MaskReduce for Struct { array.unmasked_fields().iter().cloned().collect::>(), array.struct_fields().clone(), array.len(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .map(|a| Some(a.into_array())) } diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index 910d5ebe481..78fa9af66fd 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -24,7 +24,6 @@ use crate::scalar_fn::fns::get_item::GetItem; use crate::scalar_fn::fns::mask::Mask; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; use crate::validity::Validity; -use crate::vtable::ValidityHelper; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&StructCastPushDownRule), @@ -78,11 +77,10 @@ impl ArrayParentReduceRule for StructCastPushDownRule { } let validity = if parent.options.is_nullable() { - array.validity().clone().into_nullable() + array.validity().into_nullable() } else { array .validity() - .clone() .into_non_nullable(array.len) .ok_or_else(|| vortex_err!("Failed to cast nullable struct to non-nullable"))? }; diff --git a/vortex-array/src/arrays/struct_/compute/slice.rs b/vortex-array/src/arrays/struct_/compute/slice.rs index 72edae26307..d6eea78b86a 100644 --- a/vortex-array/src/arrays/struct_/compute/slice.rs +++ b/vortex-array/src/arrays/struct_/compute/slice.rs @@ -11,7 +11,6 @@ use crate::IntoArray; use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::slice::SliceReduce; -use crate::vtable::ValidityHelper; impl SliceReduce for Struct { fn slice(array: &Self::Array, range: Range) -> VortexResult> { diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index 2809ea9e2ac..967f9eafc3f 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -12,7 +12,6 @@ use crate::arrays::dict::TakeReduce; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl TakeReduce for Struct { fn take(array: &StructArray, indices: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index 91f6b9f370b..d79ee278955 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -15,7 +15,6 @@ use crate::arrays::StructArray; use crate::builtins::ArrayBuiltins; use crate::scalar_fn::fns::zip::ZipKernel; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl ZipKernel for Struct { fn zip( @@ -39,10 +38,12 @@ impl ZipKernel for Struct { .map(|(t, f)| ArrayBuiltins::zip(mask, t.clone(), f.clone())) .collect::>>()?; - let validity = match (if_true.validity(), if_false.validity()) { - (&Validity::NonNullable, &Validity::NonNullable) => Validity::NonNullable, - (&Validity::AllValid, &Validity::AllValid) => Validity::AllValid, - (&Validity::AllInvalid, &Validity::AllInvalid) => Validity::AllInvalid, + let v1 = if_true.validity(); + let v2 = if_false.validity(); + let validity = match (&v1, &v2) { + (Validity::NonNullable, Validity::NonNullable) => Validity::NonNullable, + (Validity::AllValid, Validity::AllValid) => Validity::AllValid, + (Validity::AllInvalid, Validity::AllInvalid) => Validity::AllInvalid, (v1, v2) => { let mask_mask = mask.try_to_mask_fill_null_false(ctx)?; diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 7d348226c3d..2ace423d534 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -26,7 +26,6 @@ use crate::validity::Validity; use crate::vtable; use crate::vtable::Array; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; mod kernel; mod operations; mod validity; @@ -45,7 +44,7 @@ impl VTable for Struct { type Metadata = EmptyMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &Struct } @@ -72,7 +71,7 @@ impl VTable for Struct { for field in array.iter_unmasked_fields() { field.array_hash(state, precision); } - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &StructArray, other: &StructArray, precision: Precision) -> bool { @@ -83,7 +82,7 @@ impl VTable for Struct { .iter_unmasked_fields() .zip(other.iter_unmasked_fields()) .all(|(a, b)| a.array_eq(b, precision)) - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: &StructArray) -> usize { @@ -166,10 +165,6 @@ impl VTable for Struct { } fn with_slots(array: &mut StructArray, slots: Vec>) -> VortexResult<()> { - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/struct_/vtable/validity.rs b/vortex-array/src/arrays/struct_/vtable/validity.rs index 90dff1e618d..84378eee54b 100644 --- a/vortex-array/src/arrays/struct_/vtable/validity.rs +++ b/vortex-array/src/arrays/struct_/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + use crate::arrays::StructArray; +use crate::arrays::struct_::vtable::Struct; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for StructArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for Struct { + fn validity(array: &StructArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrays/varbin/accessor.rs b/vortex-array/src/arrays/varbin/accessor.rs index dea926defe2..990c264e4f3 100644 --- a/vortex-array/src/arrays/varbin/accessor.rs +++ b/vortex-array/src/arrays/varbin/accessor.rs @@ -8,7 +8,6 @@ use crate::accessor::ArrayAccessor; use crate::arrays::VarBinArray; use crate::match_each_integer_ptype; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl ArrayAccessor<[u8]> for VarBinArray { fn with_iterator(&self, f: F) -> R diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 3aacc926323..20e090ea728 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -19,6 +19,7 @@ use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; pub(super) const OFFSETS_SLOT: usize = 0; @@ -31,7 +32,6 @@ pub struct VarBinArray { pub(super) dtype: DType, pub(super) bytes: BufferHandle, pub(super) slots: Vec>, - pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -167,7 +167,6 @@ impl VarBinArray { dtype, bytes, slots: vec![Some(offsets), validity_slot], - validity, stats_set: Default::default(), } } @@ -266,6 +265,11 @@ impl VarBinArray { Ok(()) } + /// Reconstructs the validity from the slots. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + #[inline] pub fn offsets(&self) -> &ArrayRef { self.slots[OFFSETS_SLOT] @@ -382,10 +386,11 @@ impl VarBinArray { /// Consumes self, returning a tuple containing the `DType`, the `bytes` array, /// the `offsets` array, and the `validity`. pub fn into_parts(mut self) -> (DType, BufferHandle, ArrayRef, Validity) { + let validity = self.validity(); let offsets = self.slots[OFFSETS_SLOT] .take() .vortex_expect("VarBinArray offsets slot"); - (self.dtype, self.bytes, offsets, self.validity) + (self.dtype, self.bytes, offsets, validity) } } diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs index 4d1f46084ac..28e493a3120 100644 --- a/vortex-array/src/arrays/varbin/compute/cast.rs +++ b/vortex-array/src/arrays/varbin/compute/cast.rs @@ -9,7 +9,6 @@ use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; -use crate::vtable::ValidityHelper; impl CastReduce for VarBin { fn cast(array: &VarBinArray, dtype: &DType) -> VortexResult> { @@ -20,7 +19,6 @@ impl CastReduce for VarBin { let new_nullability = dtype.nullability(); let new_validity = array .validity() - .clone() .cast_nullability(new_nullability, array.len())?; let new_dtype = array.dtype().with_nullability(new_nullability); Ok(Some( diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index 07af710f762..b65d652680e 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -28,7 +28,6 @@ use crate::match_each_integer_ptype; use crate::scalar_fn::fns::binary::CompareKernel; use crate::scalar_fn::fns::operators::CompareOperator; use crate::scalar_fn::fns::operators::Operator; -use crate::vtable::ValidityHelper; // This implementation exists so we can have custom translation of RHS to arrow that's not the same as IntoCanonical impl CompareKernel for VarBin { @@ -75,9 +74,7 @@ impl CompareKernel for VarBin { return Ok(Some( BoolArray::new( buffer, - lhs.validity() - .clone() - .union_nullability(rhs.dtype().nullability()), + lhs.validity().union_nullability(rhs.dtype().nullability()), ) .into_array(), )); diff --git a/vortex-array/src/arrays/varbin/compute/filter.rs b/vortex-array/src/arrays/varbin/compute/filter.rs index ba41c60f273..83ced2e0377 100644 --- a/vortex-array/src/arrays/varbin/compute/filter.rs +++ b/vortex-array/src/arrays/varbin/compute/filter.rs @@ -23,7 +23,6 @@ use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::match_each_integer_ptype; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl FilterKernel for VarBin { fn filter( @@ -166,7 +165,7 @@ fn filter_select_var_bin_by_index( offsets.as_slice::(), values.bytes().as_slice(), mask_indices, - values.validity().clone(), + values.validity(), selection_count, ) }) diff --git a/vortex-array/src/arrays/varbin/compute/mask.rs b/vortex-array/src/arrays/varbin/compute/mask.rs index 3169adaf0ff..3a796be28fe 100644 --- a/vortex-array/src/arrays/varbin/compute/mask.rs +++ b/vortex-array/src/arrays/varbin/compute/mask.rs @@ -9,7 +9,6 @@ use crate::arrays::VarBin; use crate::arrays::VarBinArray; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for VarBin { fn mask(array: &VarBinArray, mask: &ArrayRef) -> VortexResult> { @@ -18,10 +17,7 @@ impl MaskReduce for VarBin { array.offsets().clone(), array.bytes().clone(), array.dtype().as_nullable(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, )? .into_array(), )) diff --git a/vortex-array/src/arrays/varbin/compute/slice.rs b/vortex-array/src/arrays/varbin/compute/slice.rs index 5c14088c324..6575968a976 100644 --- a/vortex-array/src/arrays/varbin/compute/slice.rs +++ b/vortex-array/src/arrays/varbin/compute/slice.rs @@ -24,7 +24,7 @@ impl VarBin { array.offsets().slice(range.start..range.end + 1)?, array.bytes_handle().clone(), array.dtype().clone(), - array.validity()?.slice(range)?, + array.validity().slice(range)?, ) .into_array() }) diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 7c123a0ac02..0ed565a2587 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -19,7 +19,6 @@ use crate::SerializeMetadata; use crate::arrays::VarBinArray; use crate::arrays::varbin::array::NUM_SLOTS; use crate::arrays::varbin::array::SLOT_NAMES; -use crate::arrays::varbin::array::VALIDITY_SLOT; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::Nullability; @@ -30,7 +29,6 @@ use crate::vtable; use crate::vtable::Array; use crate::vtable::ArrayId; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; mod canonical; mod kernel; mod operations; @@ -60,7 +58,7 @@ impl VTable for VarBin { type Metadata = ProstMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &VarBin } @@ -85,14 +83,14 @@ impl VTable for VarBin { array.dtype.hash(state); array.bytes().array_hash(state, precision); array.offsets().array_hash(state, precision); - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &VarBinArray, other: &VarBinArray, precision: Precision) -> bool { array.dtype == other.dtype && array.bytes().array_eq(other.bytes(), precision) && array.offsets().array_eq(other.offsets(), precision) - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: &VarBinArray) -> usize { @@ -181,10 +179,6 @@ impl VTable for VarBin { NUM_SLOTS, slots.len() ); - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/varbin/vtable/validity.rs b/vortex-array/src/arrays/varbin/vtable/validity.rs index f436968206c..7202315448d 100644 --- a/vortex-array/src/arrays/varbin/vtable/validity.rs +++ b/vortex-array/src/arrays/varbin/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + use crate::arrays::VarBinArray; +use crate::arrays::varbin::vtable::VarBin; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for VarBinArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for VarBin { + fn validity(array: &VarBinArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrays/varbinview/accessor.rs b/vortex-array/src/arrays/varbinview/accessor.rs index d494c4711a7..912d7ee8de3 100644 --- a/vortex-array/src/arrays/varbinview/accessor.rs +++ b/vortex-array/src/arrays/varbinview/accessor.rs @@ -7,7 +7,6 @@ use crate::ToCanonical; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinViewArray; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl ArrayAccessor<[u8]> for VarBinViewArray { fn with_iterator FnOnce(&mut dyn Iterator>) -> R, R>( diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 32451b91cd5..521ad798f6c 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -22,6 +22,7 @@ use crate::dtype::DType; use crate::dtype::Nullability; use crate::stats::ArrayStats; use crate::validity::Validity; +use crate::vtable::child_to_validity; use crate::vtable::validity_to_child; pub(super) const VALIDITY_SLOT: usize = 0; @@ -93,7 +94,6 @@ pub struct VarBinViewArray { pub(super) dtype: DType, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, - pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } @@ -261,7 +261,6 @@ impl VarBinViewArray { views, buffers, dtype, - validity, stats_set: Default::default(), } } @@ -355,13 +354,19 @@ impl VarBinViewArray { Ok(()) } + /// Reconstructs the validity from the slots. + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) + } + /// Splits the array into owned parts pub fn into_parts(self) -> VarBinViewArrayParts { + let validity = self.validity(); VarBinViewArrayParts { dtype: self.dtype, buffers: self.buffers, views: self.views, - validity: self.validity, + validity, } } diff --git a/vortex-array/src/arrays/varbinview/compute/cast.rs b/vortex-array/src/arrays/varbinview/compute/cast.rs index 5486e576740..b15c2ed11ba 100644 --- a/vortex-array/src/arrays/varbinview/compute/cast.rs +++ b/vortex-array/src/arrays/varbinview/compute/cast.rs @@ -9,7 +9,6 @@ use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; -use crate::vtable::ValidityHelper; impl CastReduce for VarBinView { fn cast(array: &VarBinViewArray, dtype: &DType) -> VortexResult> { @@ -20,7 +19,6 @@ impl CastReduce for VarBinView { let new_nullability = dtype.nullability(); let new_validity = array .validity() - .clone() .cast_nullability(new_nullability, array.len())?; let new_dtype = array.dtype().with_nullability(new_nullability); diff --git a/vortex-array/src/arrays/varbinview/compute/mask.rs b/vortex-array/src/arrays/varbinview/compute/mask.rs index dae65ab40bd..ac6d64779c5 100644 --- a/vortex-array/src/arrays/varbinview/compute/mask.rs +++ b/vortex-array/src/arrays/varbinview/compute/mask.rs @@ -9,7 +9,6 @@ use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::scalar_fn::fns::mask::MaskReduce; use crate::validity::Validity; -use crate::vtable::ValidityHelper; impl MaskReduce for VarBinView { fn mask(array: &VarBinViewArray, mask: &ArrayRef) -> VortexResult> { @@ -20,10 +19,7 @@ impl MaskReduce for VarBinView { array.views_handle().clone(), array.buffers().clone(), array.dtype().as_nullable(), - array - .validity() - .clone() - .and(Validity::Array(mask.clone()))?, + array.validity().and(Validity::Array(mask.clone()))?, ) .into_array(), )) diff --git a/vortex-array/src/arrays/varbinview/compute/slice.rs b/vortex-array/src/arrays/varbinview/compute/slice.rs index 02582841601..2bc4c84b2e7 100644 --- a/vortex-array/src/arrays/varbinview/compute/slice.rs +++ b/vortex-array/src/arrays/varbinview/compute/slice.rs @@ -22,7 +22,7 @@ impl SliceReduce for VarBinView { .slice_typed::(range.clone()), Arc::clone(array.buffers()), array.dtype().clone(), - array.validity()?.slice(range)?, + array.validity().slice(range)?, ) .into_array(), )) diff --git a/vortex-array/src/arrays/varbinview/compute/take.rs b/vortex-array/src/arrays/varbinview/compute/take.rs index b3620aaaaf0..388124fb007 100644 --- a/vortex-array/src/arrays/varbinview/compute/take.rs +++ b/vortex-array/src/arrays/varbinview/compute/take.rs @@ -19,7 +19,6 @@ use crate::arrays::varbinview::BinaryView; use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::match_each_integer_ptype; -use crate::vtable::ValidityHelper; impl TakeExecute for VarBinView { /// Take involves creating a new array that references the old array, just with the given set of views. diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 13793db4e6c..7c7f809f50d 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -23,7 +23,6 @@ use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::BinaryView; use crate::arrays::varbinview::array::NUM_SLOTS; use crate::arrays::varbinview::array::SLOT_NAMES; -use crate::arrays::varbinview::array::VALIDITY_SLOT; use crate::arrays::varbinview::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -36,7 +35,6 @@ use crate::vtable; use crate::vtable::Array; use crate::vtable::ArrayId; use crate::vtable::VTable; -use crate::vtable::ValidityVTableFromValidityHelper; mod kernel; mod operations; mod validity; @@ -54,7 +52,7 @@ impl VTable for VarBinView { type Metadata = EmptyMetadata; type OperationsVTable = Self; - type ValidityVTable = ValidityVTableFromValidityHelper; + type ValidityVTable = Self; fn vtable(_array: &Self::Array) -> &Self { &VarBinView } @@ -85,7 +83,7 @@ impl VTable for VarBinView { buffer.array_hash(state, precision); } array.views.array_hash(state, precision); - array.validity.array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &VarBinViewArray, other: &VarBinViewArray, precision: Precision) -> bool { @@ -97,7 +95,7 @@ impl VTable for VarBinView { .zip(other.buffers.iter()) .all(|(a, b)| a.array_eq(b, precision)) && array.views.array_eq(&other.views, precision) - && array.validity.array_eq(&other.validity, precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(array: &VarBinViewArray) -> usize { @@ -210,10 +208,6 @@ impl VTable for VarBinView { NUM_SLOTS, slots.len() ); - array.validity = match &slots[VALIDITY_SLOT] { - Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.dtype.nullability()), - }; array.slots = slots; Ok(()) } diff --git a/vortex-array/src/arrays/varbinview/vtable/validity.rs b/vortex-array/src/arrays/varbinview/vtable/validity.rs index bc8410ba5c0..5e42941c247 100644 --- a/vortex-array/src/arrays/varbinview/vtable/validity.rs +++ b/vortex-array/src/arrays/varbinview/vtable/validity.rs @@ -1,12 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexResult; + use crate::arrays::VarBinViewArray; +use crate::arrays::varbinview::vtable::VarBinView; use crate::validity::Validity; -use crate::vtable::ValidityHelper; +use crate::vtable::ValidityVTable; -impl ValidityHelper for VarBinViewArray { - fn validity(&self) -> &Validity { - &self.validity +impl ValidityVTable for VarBinView { + fn validity(array: &VarBinViewArray) -> VortexResult { + Ok(array.validity()) } } diff --git a/vortex-array/src/arrow/executor/byte.rs b/vortex-array/src/arrow/executor/byte.rs index 4fbf13b1544..434758b509d 100644 --- a/vortex-array/src/arrow/executor/byte.rs +++ b/vortex-array/src/arrow/executor/byte.rs @@ -23,7 +23,6 @@ use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; -use crate::vtable::ValidityHelper; /// Convert a Vortex array into an Arrow GenericBinaryArray. pub(super) fn to_arrow_byte_array( @@ -68,7 +67,7 @@ where let data = array.bytes().clone().into_arrow_buffer(); - let null_buffer = to_arrow_null_buffer(array.validity().clone(), array.len(), ctx)?; + let null_buffer = to_arrow_null_buffer(array.validity(), array.len(), ctx)?; Ok(Arc::new(unsafe { GenericByteArray::::new_unchecked(offsets, data, null_buffer) })) diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-array/src/arrow/executor/byte_view.rs index 0e6b4923325..eb0995fb95c 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-array/src/arrow/executor/byte_view.rs @@ -18,7 +18,6 @@ use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::arrow::FromArrowType; -use crate::vtable::ValidityHelper; /// Convert a canonical VarBinViewArray directly to Arrow. pub fn canonical_varbinview_to_arrow( @@ -50,7 +49,7 @@ pub fn execute_varbinview_to_arrow( .iter() .map(|buffer| buffer.as_host().clone().into_arrow_buffer()) .collect(); - let nulls = to_arrow_null_buffer(array.validity().clone(), array.len(), ctx)?; + let nulls = to_arrow_null_buffer(array.validity(), array.len(), ctx)?; // SAFETY: our own VarBinView array is considered safe. Ok(Arc::new(unsafe { diff --git a/vortex-array/src/arrow/executor/fixed_size_list.rs b/vortex-array/src/arrow/executor/fixed_size_list.rs index 26f4d96ff6b..92546322cf6 100644 --- a/vortex-array/src/arrow/executor/fixed_size_list.rs +++ b/vortex-array/src/arrow/executor/fixed_size_list.rs @@ -53,7 +53,7 @@ fn list_to_list( "Cannot convert FixedSizeListArray to non-nullable Arrow array when elements are nullable" ); - let null_buffer = to_arrow_null_buffer(array.validity().clone(), array.len(), ctx)?; + let null_buffer = to_arrow_null_buffer(array.validity(), array.len(), ctx)?; Ok(Arc::new( arrow_array::FixedSizeListArray::try_new_with_length( diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index dc2d9bd0e25..a4996ce43bb 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -104,7 +104,7 @@ fn list_to_list( "Cannot convert to non-nullable Arrow array with null elements" ); - let null_buffer = to_arrow_null_buffer(array.validity().clone(), array.len(), ctx)?; + let null_buffer = to_arrow_null_buffer(array.validity(), array.len(), ctx)?; // TODO(ngates): use new_unchecked when it is added to arrow-rs. Ok(Arc::new(GenericListArray::::new( diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index 7e4b979fcda..8844f49edef 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -163,7 +163,6 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::vtable::ValidityHelper; fn make_opt_bool_chunks(len: usize, chunk_count: usize) -> ArrayRef { let mut rng = StdRng::seed_from_u64(0); @@ -200,7 +199,7 @@ mod tests { assert!( canon_into .validity() - .mask_eq(into_canon.validity(), &mut ctx)? + .mask_eq(&into_canon.validity(), &mut ctx)? ); assert_eq!(canon_into.to_bit_buffer(), into_canon.to_bit_buffer()); Ok(()) diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 37c44b3cada..efdd44bd739 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -471,7 +471,7 @@ mod tests { assert!( actual .validity() - .mask_eq(expected.validity(), &mut ctx) + .mask_eq(&expected.validity(), &mut ctx) .unwrap(), ); } diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index f486cadc83f..6c42a7f4011 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -619,7 +619,6 @@ mod tests { // values[2] might be any value since it's null. // Check validity - first two should be valid, third should be null. - use crate::vtable::ValidityHelper; assert!(array.validity().is_valid(0).unwrap()); assert!(array.validity().is_valid(1).unwrap()); assert!(!array.validity().is_valid(2).unwrap()); diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index e57ec9c64ec..2c2300ae2b4 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -45,7 +45,6 @@ use crate::search_sorted::SearchResult; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedSide; use crate::validity::Validity; -use crate::vtable::ValidityHelper; /// One patch index offset is stored for each chunk. /// This allows for constant time patch index lookups. @@ -913,7 +912,7 @@ impl Patches { patch_indices_slice, self.offset, patch_values_slice, - patches_validity, + &patches_validity, ctx, ); } diff --git a/vortex-array/src/scalar/convert/from_scalar.rs b/vortex-array/src/scalar/convert/from_scalar.rs index 4f5dbc98b98..32753dea1dc 100644 --- a/vortex-array/src/scalar/convert/from_scalar.rs +++ b/vortex-array/src/scalar/convert/from_scalar.rs @@ -113,7 +113,10 @@ impl TryFrom<&Scalar> for bool { type Error = VortexError; fn try_from(value: &Scalar) -> VortexResult { - >::try_from(value)? + value + .as_bool_opt() + .ok_or_else(|| vortex_err!("Expected bool scalar, found {}", value.dtype()))? + .value() .ok_or_else(|| vortex_err!("Can't extract present value from null scalar")) } } diff --git a/vortex-array/src/scalar_fn/fns/get_item.rs b/vortex-array/src/scalar_fn/fns/get_item.rs index 1e0469bae27..cc87765e3a2 100644 --- a/vortex-array/src/scalar_fn/fns/get_item.rs +++ b/vortex-array/src/scalar_fn/fns/get_item.rs @@ -116,7 +116,7 @@ impl ScalarFnVTable for GetItem { match input.dtype().nullability() { Nullability::NonNullable => Ok(field), - Nullability::Nullable => field.mask(input.validity()?.to_array(input.len())), + Nullability::Nullable => field.mask(input.validity().to_array(input.len())), } } diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index dd0b2d46918..3ed85bd7243 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -344,7 +344,7 @@ fn list_contains_scalar( Ok(BoolArray::new( list_matches, - list_array.validity().clone().union_nullability(nullability), + list_array.validity().union_nullability(nullability), ) .into_array()) } @@ -434,11 +434,7 @@ fn list_is_not_empty( }); // Copy over the validity mask from the input. - Ok(BoolArray::new( - buffer, - list_array.validity().clone().union_nullability(nullability), - ) - .into_array()) + Ok(BoolArray::new(buffer, list_array.validity().union_nullability(nullability)).into_array()) } #[cfg(test)] diff --git a/vortex-array/src/scalar_fn/fns/not/mod.rs b/vortex-array/src/scalar_fn/fns/not/mod.rs index bdeae78cc66..6f52e4a77ff 100644 --- a/vortex-array/src/scalar_fn/fns/not/mod.rs +++ b/vortex-array/src/scalar_fn/fns/not/mod.rs @@ -103,7 +103,7 @@ impl ScalarFnVTable for Not { // For boolean array if let Some(bool) = child.as_opt::() { - return Ok(BoolArray::new(!bool.to_bit_buffer(), bool.validity()?).into_array()); + return Ok(BoolArray::new(!bool.to_bit_buffer(), bool.validity()).into_array()); } // Otherwise, execute and try again diff --git a/vortex-array/src/scalar_fn/fns/pack.rs b/vortex-array/src/scalar_fn/fns/pack.rs index 42741d2476f..37a00e61aeb 100644 --- a/vortex-array/src/scalar_fn/fns/pack.rs +++ b/vortex-array/src/scalar_fn/fns/pack.rs @@ -177,7 +177,6 @@ mod tests { use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::fns::pack::StructArray; use crate::validity::Validity; - use crate::vtable::ValidityHelper; fn test_array() -> ArrayRef { StructArray::from_fields(&[ diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index ee4f03496e2..3dd6695ea59 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -29,11 +29,14 @@ use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; +use crate::arrays::constant::Constant; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; +use crate::dtype::Nullability; use crate::executor::ExecutionCtx; use crate::patches::Patches; +use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; use crate::validity::Validity; @@ -243,6 +246,30 @@ pub fn validity_to_child(validity: &Validity, len: usize) -> Option { } } +/// Reconstruct a [`Validity`] from an optional child array and nullability. +/// +/// This is the inverse of [`validity_to_child`]. +#[inline] +pub fn child_to_validity(child: &Option, nullability: Nullability) -> Validity { + match child { + Some(arr) => { + // Detect constant bool arrays created by validity_to_child. + // Use direct ScalarValue matching to avoid expensive scalar conversion. + if let Some(c) = arr.as_opt::() + && let Some(ScalarValue::Bool(val)) = c.scalar().value() + { + return if *val { + Validity::AllValid + } else { + Validity::AllInvalid + }; + } + Validity::Array(arr.clone()) + } + None => Validity::from(nullability), + } +} + /// Returns 1 if validity produces a child, 0 otherwise. #[inline] pub fn validity_nchildren(validity: &Validity) -> usize { diff --git a/vortex-array/src/vtable/validity.rs b/vortex-array/src/vtable/validity.rs index ed2929e958e..e39ddac53be 100644 --- a/vortex-array/src/vtable/validity.rs +++ b/vortex-array/src/vtable/validity.rs @@ -22,7 +22,7 @@ pub struct ValidityVTableFromValidityHelper; /// Expose validity held as a child array. pub trait ValidityHelper { - fn validity(&self) -> &Validity; + fn validity(&self) -> Validity; } impl ValidityVTable for ValidityVTableFromValidityHelper @@ -30,7 +30,7 @@ where V::Array: ValidityHelper, { fn validity(array: &V::Array) -> VortexResult { - Ok(array.validity().clone()) + Ok(array.validity()) } } diff --git a/vortex-btrblocks/public-api.lock b/vortex-btrblocks/public-api.lock index 7742516a2c0..eb9643db354 100644 --- a/vortex-btrblocks/public-api.lock +++ b/vortex-btrblocks/public-api.lock @@ -632,6 +632,6 @@ pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::fmt(&self, f: &mut core::fm pub const vortex_btrblocks::ALL_SCHEMES: &[&dyn vortex_compressor::scheme::Scheme] -pub fn vortex_btrblocks::compress_patches(patches: &vortex_array::patches::Patches) -> vortex_error::VortexResult +pub fn vortex_btrblocks::compress_patches(patches: vortex_array::patches::Patches) -> vortex_error::VortexResult pub fn vortex_btrblocks::default_excluded() -> vortex_utils::aliases::hash_set::HashSet diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index dcbf74c6f10..8fd21aa75cd 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -10,7 +10,6 @@ use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::decimal::narrowed_decimal; use vortex_array::dtype::DecimalType; -use vortex_array::vtable::ValidityHelper; use vortex_decimal_byte_parts::DecimalBytePartsArray; use vortex_error::VortexResult; @@ -63,10 +62,10 @@ impl Scheme for DecimalScheme { let decimal = narrowed_decimal(decimal); let validity = decimal.validity(); let prim = match decimal.values_type() { - DecimalType::I8 => PrimitiveArray::new(decimal.buffer::(), validity.clone()), - DecimalType::I16 => PrimitiveArray::new(decimal.buffer::(), validity.clone()), - DecimalType::I32 => PrimitiveArray::new(decimal.buffer::(), validity.clone()), - DecimalType::I64 => PrimitiveArray::new(decimal.buffer::(), validity.clone()), + DecimalType::I8 => PrimitiveArray::new(decimal.buffer::(), validity), + DecimalType::I16 => PrimitiveArray::new(decimal.buffer::(), validity), + DecimalType::I32 => PrimitiveArray::new(decimal.buffer::(), validity), + DecimalType::I64 => PrimitiveArray::new(decimal.buffer::(), validity), _ => return Ok(decimal.into_array()), }; diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 3609c77a90c..e3eb7b7649b 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -786,7 +786,8 @@ mod tests { false, false, false, false, false, false, false, false, false, false, true, ]), ); - let validity = array.validity()?; + + let validity = array.validity(); let btr = BtrBlocksCompressor::default(); let compressed = btr.compress(&array.into_array())?; diff --git a/vortex-btrblocks/src/schemes/patches.rs b/vortex-btrblocks/src/schemes/patches.rs index 29612b56a8c..38ec1cf9068 100644 --- a/vortex-btrblocks/src/schemes/patches.rs +++ b/vortex-btrblocks/src/schemes/patches.rs @@ -11,7 +11,7 @@ use vortex_error::VortexError; use vortex_error::VortexResult; /// Compresses the given patches by downscaling integers and checking for constant values. -pub fn compress_patches(patches: &Patches) -> VortexResult { +pub fn compress_patches(patches: Patches) -> VortexResult { // Downscale the patch indices. let indices = patches.indices().to_primitive().narrow()?.into_array(); diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 0840a2dfae2..fbcb771e9b5 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -8,7 +8,6 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::VarBinArray; -use vortex_array::vtable::ValidityHelper; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; use vortex_error::VortexResult; @@ -100,7 +99,7 @@ impl Scheme for FSSTScheme { compressed_codes_offsets, fsst.codes().bytes().clone(), fsst.codes().dtype().clone(), - fsst.codes().validity().clone(), + fsst.codes().validity(), )?; let fsst = FSSTArray::try_new( diff --git a/vortex-compressor/src/builtins/constant.rs b/vortex-compressor/src/builtins/constant.rs index ac38aee732c..53300d49703 100644 --- a/vortex-compressor/src/builtins/constant.rs +++ b/vortex-compressor/src/builtins/constant.rs @@ -11,7 +11,6 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::scalar::Scalar; -use vortex_array::vtable::ValidityHelper; use vortex_error::VortexResult; use super::is_bool; @@ -231,10 +230,7 @@ impl Scheme for StringConstantScheme { let scalar = stats.source().scalar_at(idx)?; let const_arr = ConstantArray::new(scalar, stats.source().len()).into_array(); if !stats.source().all_valid()? { - Ok( - MaskedArray::try_new(const_arr, stats.source().validity().clone())? - .into_array(), - ) + Ok(MaskedArray::try_new(const_arr, stats.source().validity())?.into_array()) } else { Ok(const_arr) } @@ -257,7 +253,7 @@ fn compress_constant_primitive(source: &PrimitiveArray) -> VortexResult { let len = varbinview.len(); - check_validity_empty(varbinview.validity())?; + check_validity_empty(&varbinview.validity())?; let BinaryParts { offsets, bytes } = copy_varbinview_to_varbin(varbinview, ctx).await?; diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index b9b2c803331..6b2e00a1580 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -324,7 +324,7 @@ mod tests { Ok(PrimitiveArray::from_byte_buffer( prim.buffer_handle().try_to_host_sync()?, prim.ptype(), - prim.validity()?, + prim.validity(), )) } @@ -655,7 +655,7 @@ mod tests { BufferHandle::new_host(decimal.buffer_handle().try_to_host_sync()?), decimal.values_type(), decimal.decimal_dtype(), - decimal.validity()?, + decimal.validity(), )) } diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 663ca621860..dd107e87154 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -15,7 +15,6 @@ use tracing::instrument; use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::patches::Patches; use vortex::array::validity::Validity; -use vortex::array::vtable::ValidityHelper; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::error::vortex_ensure; diff --git a/vortex-duckdb/src/exporter/bool.rs b/vortex-duckdb/src/exporter/bool.rs index 75dfa195bc9..0de95415a40 100644 --- a/vortex-duckdb/src/exporter/bool.rs +++ b/vortex-duckdb/src/exporter/bool.rs @@ -23,8 +23,7 @@ pub(crate) fn new_exporter( ) -> VortexResult> { let len = array.len(); let bits = array.to_bit_buffer(); - let validity = array.validity()?.to_array(len).execute::(ctx)?; - + let validity = array.validity().to_array(len).execute::(ctx)?; if validity.all_false() { return Ok(all_invalid::new_exporter(len, &LogicalType::bool())); } diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs index 144abfb099a..97fd0edf861 100644 --- a/vortex-duckdb/src/exporter/mod.rs +++ b/vortex-duckdb/src/exporter/mod.rs @@ -31,7 +31,6 @@ use vortex::array::arrays::Dict; use vortex::array::arrays::List; use vortex::array::arrays::StructArray; use vortex::array::arrays::TemporalArray; -use vortex::array::vtable::ValidityHelper; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::Sequence; use vortex::error::VortexResult; diff --git a/vortex-duckdb/src/exporter/primitive.rs b/vortex-duckdb/src/exporter/primitive.rs index 5e4a3f7e121..c5167880b59 100644 --- a/vortex-duckdb/src/exporter/primitive.rs +++ b/vortex-duckdb/src/exporter/primitive.rs @@ -6,7 +6,6 @@ use std::marker::PhantomData; use vortex::array::ExecutionCtx; use vortex::array::arrays::PrimitiveArray; use vortex::array::match_each_native_ptype; -use vortex::array::vtable::ValidityHelper; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index 1394305ac3e..1b6fc51b44b 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -30,7 +30,6 @@ use vortex_array::expr::transform::replace; use vortex_array::expr::transform::replace_root_fields; use vortex_array::scalar_fn::fns::merge::Merge; use vortex_array::scalar_fn::fns::pack::Pack; -use vortex_array::vtable::ValidityHelper; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -371,7 +370,7 @@ impl LayoutReader for StructReader { struct_array.names().clone(), masked_fields, struct_array.len(), - struct_array.validity().clone(), + struct_array.validity(), )? .into_array()) } else { diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 4d88546d2df..d13049c780f 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -23,7 +23,6 @@ use vortex::array::arrays::TemporalArray; use vortex::array::arrays::VarBinArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::builtins::ArrayBuiltins; -use vortex::array::vtable::ValidityHelper; use vortex::dtype::DType; use vortex::dtype::PType; use vortex::encodings::alp::alp_encode; @@ -109,7 +108,7 @@ mod setup { vortex::encodings::alp::ALPArray::try_new( for_with_bp.into_array(), alp_compressed.exponents(), - alp_compressed.patches().cloned(), + alp_compressed.patches(), ) .unwrap() .into_array() @@ -253,7 +252,7 @@ mod setup { offsets_bp.into_array(), codes.bytes().clone(), codes.dtype().clone(), - codes.validity().clone(), + codes.validity(), ) .unwrap(); diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index ab22ea36f4e..318d19a052d 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -196,7 +196,6 @@ mod test { use vortex_array::expr::select; use vortex_array::stream::ArrayStreamExt; use vortex_array::validity::Validity; - use vortex_array::vtable::ValidityHelper; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_file::OpenOptionsSessionExt; @@ -339,7 +338,7 @@ mod test { assert!( recovered_primitive .validity() - .mask_eq(array.validity(), &mut ctx)? + .mask_eq(&array.validity(), &mut ctx)? ); assert_eq!( recovered_primitive.to_buffer::(),