diff --git a/arrow-cast/src/cast.rs b/arrow-cast/src/cast.rs index 23be8839593c..cb198521a31a 100644 --- a/arrow-cast/src/cast.rs +++ b/arrow-cast/src/cast.rs @@ -1052,17 +1052,20 @@ pub fn cast_with_options( Int64 => cast_numeric_to_string::(array), Float32 => cast_numeric_to_string::(array), Float64 => cast_numeric_to_string::(array), - Timestamp(TimeUnit::Nanosecond, tz) => { - cast_timestamp_to_string::(array, tz) - } - Timestamp(TimeUnit::Microsecond, tz) => { - cast_timestamp_to_string::(array, tz) - } - Timestamp(TimeUnit::Millisecond, tz) => { - cast_timestamp_to_string::(array, tz) - } + Timestamp(TimeUnit::Nanosecond, tz) => cast_timestamp_to_string::< + TimestampNanosecondType, + i32, + >(array, tz.as_ref()), + Timestamp(TimeUnit::Microsecond, tz) => cast_timestamp_to_string::< + TimestampMicrosecondType, + i32, + >(array, tz.as_ref()), + Timestamp(TimeUnit::Millisecond, tz) => cast_timestamp_to_string::< + TimestampMillisecondType, + i32, + >(array, tz.as_ref()), Timestamp(TimeUnit::Second, tz) => { - cast_timestamp_to_string::(array, tz) + cast_timestamp_to_string::(array, tz.as_ref()) } Date32 => cast_date32_to_string::(array), Date64 => cast_date64_to_string::(array), @@ -1106,17 +1109,20 @@ pub fn cast_with_options( Int64 => cast_numeric_to_string::(array), Float32 => cast_numeric_to_string::(array), Float64 => cast_numeric_to_string::(array), - Timestamp(TimeUnit::Nanosecond, tz) => { - cast_timestamp_to_string::(array, tz) - } - Timestamp(TimeUnit::Microsecond, tz) => { - cast_timestamp_to_string::(array, tz) - } - Timestamp(TimeUnit::Millisecond, tz) => { - cast_timestamp_to_string::(array, tz) - } + Timestamp(TimeUnit::Nanosecond, tz) => cast_timestamp_to_string::< + TimestampNanosecondType, + i64, + >(array, tz.as_ref()), + Timestamp(TimeUnit::Microsecond, tz) => cast_timestamp_to_string::< + TimestampMicrosecondType, + i64, + >(array, tz.as_ref()), + Timestamp(TimeUnit::Millisecond, tz) => cast_timestamp_to_string::< + TimestampMillisecondType, + i64, + >(array, tz.as_ref()), Timestamp(TimeUnit::Second, tz) => { - cast_timestamp_to_string::(array, tz) + cast_timestamp_to_string::(array, tz.as_ref()) } Date32 => cast_date32_to_string::(array), Date64 => cast_date64_to_string::(array), @@ -2472,7 +2478,7 @@ where /// Cast timestamp types to Utf8/LargeUtf8 fn cast_timestamp_to_string( array: &ArrayRef, - tz: &Option, + tz: Option<&String>, ) -> Result where T: ArrowTemporalType + ArrowPrimitiveType, diff --git a/arrow-ipc/src/compression.rs b/arrow-ipc/src/compression.rs index 6349ac232431..f64d14441cb1 100644 --- a/arrow-ipc/src/compression.rs +++ b/arrow-ipc/src/compression.rs @@ -22,8 +22,8 @@ use arrow_schema::ArrowError; const LENGTH_NO_COMPRESSED_DATA: i64 = -1; const LENGTH_OF_PREFIX_DATA: i64 = 8; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Represents compressing a ipc stream using a particular compression algorithm +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CompressionCodec { Lz4Frame, Zstd, diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs index 032783deed72..5f188fe1a9fc 100644 --- a/arrow-ipc/src/writer.rs +++ b/arrow-ipc/src/writer.rs @@ -377,7 +377,7 @@ impl IpcDataGenerator { offset, array.len(), array.null_count(), - &compression_codec, + compression_codec, write_options, )?; } @@ -452,7 +452,7 @@ impl IpcDataGenerator { 0, array_data.len(), array_data.null_count(), - &compression_codec, + compression_codec, write_options, )?; @@ -1058,7 +1058,7 @@ fn write_array_data( offset: i64, num_rows: usize, null_count: usize, - compression_codec: &Option, + compression_codec: Option, write_options: &IpcWriteOptions, ) -> Result { let mut offset = offset; @@ -1234,7 +1234,7 @@ fn write_buffer( buffers: &mut Vec, // output buffer descriptors arrow_data: &mut Vec, // output stream offset: i64, // current output stream offset - compression_codec: &Option, + compression_codec: Option, ) -> Result { let len: i64 = match compression_codec { Some(compressor) => compressor.compress_to_vec(buffer, arrow_data)?, diff --git a/parquet/src/arrow/arrow_reader/mod.rs b/parquet/src/arrow/arrow_reader/mod.rs index da4b56237e14..e89ddaffe833 100644 --- a/parquet/src/arrow/arrow_reader/mod.rs +++ b/parquet/src/arrow/arrow_reader/mod.rs @@ -1567,7 +1567,8 @@ mod tests { let expected_data = match opts.row_selections { Some((selections, row_count)) => { - let mut without_skip_data = gen_expected_data::(&def_levels, &values); + let mut without_skip_data = + gen_expected_data::(def_levels.as_ref(), &values); let mut skip_data: Vec> = vec![]; let dequeue: VecDeque = selections.clone().into(); @@ -1585,7 +1586,7 @@ mod tests { } None => { //get flatten table data - let expected_data = gen_expected_data::(&def_levels, &values); + let expected_data = gen_expected_data::(def_levels.as_ref(), &values); assert_eq!(expected_data.len(), opts.num_rows * opts.num_row_groups); expected_data } @@ -1654,7 +1655,7 @@ mod tests { } fn gen_expected_data( - def_levels: &Option>>, + def_levels: Option<&Vec>>, values: &[Vec], ) -> Vec> { let data: Vec> = match def_levels { diff --git a/parquet/src/arrow/async_reader.rs b/parquet/src/arrow/async_reader.rs index e182cccbcea3..7602d54a5107 100644 --- a/parquet/src/arrow/async_reader.rs +++ b/parquet/src/arrow/async_reader.rs @@ -903,10 +903,8 @@ mod tests { // Check offset indexes are present for all columns for rg in metadata_with_index.row_groups() { - let page_locations = rg - .page_offset_index() - .as_ref() - .expect("expected page offset index"); + let page_locations = + rg.page_offset_index().expect("expected page offset index"); assert_eq!(page_locations.len(), rg.columns().len()) } diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs index 40f8c99403f0..1010dc156a02 100644 --- a/parquet/src/column/writer/mod.rs +++ b/parquet/src/column/writer/mod.rs @@ -601,7 +601,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { } /// Update the column index and offset index when adding the data page - fn update_column_offset_index(&mut self, page_statistics: &Option) { + fn update_column_offset_index(&mut self, page_statistics: Option<&Statistics>) { // update the column index let null_page = (self.page_metrics.num_buffered_rows as u64) == self.page_metrics.num_page_nulls; @@ -664,7 +664,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { }; // update column and offset index - self.update_column_offset_index(&page_statistics); + self.update_column_offset_index(page_statistics.as_ref()); let compressed_page = match self.props.writer_version() { WriterVersion::PARQUET_1_0 => { diff --git a/parquet/src/file/metadata.rs b/parquet/src/file/metadata.rs index 2ba50fa31a1e..51a5264e3cf1 100644 --- a/parquet/src/file/metadata.rs +++ b/parquet/src/file/metadata.rs @@ -277,8 +277,8 @@ impl RowGroupMetaData { } /// Returns reference of page offset index of all column in this row group. - pub fn page_offset_index(&self) -> &Option>> { - &self.page_offset_index + pub fn page_offset_index(&self) -> Option<&Vec>> { + self.page_offset_index.as_ref() } /// Returns reference to a schema descriptor. diff --git a/parquet/tests/arrow_writer_layout.rs b/parquet/tests/arrow_writer_layout.rs index 5744de35e337..bf24950e99c2 100644 --- a/parquet/tests/arrow_writer_layout.rs +++ b/parquet/tests/arrow_writer_layout.rs @@ -81,7 +81,7 @@ fn assert_layout(file_reader: &Bytes, meta: &ParquetMetaData, layout: &Layout) { for (row_group, row_group_layout) in meta.row_groups().iter().zip(&layout.row_groups) { // Check against offset index - let offset_index = row_group.page_offset_index().as_ref().unwrap(); + let offset_index = row_group.page_offset_index().unwrap(); assert_eq!(offset_index.len(), row_group_layout.columns.len()); for (column_index, column_layout) in diff --git a/parquet_derive/src/parquet_field.rs b/parquet_derive/src/parquet_field.rs index 06bcc0aca924..48b6d3ac41b8 100644 --- a/parquet_derive/src/parquet_field.rs +++ b/parquet_derive/src/parquet_field.rs @@ -672,8 +672,8 @@ mod test { let struct_def: proc_macro2::TokenStream = quote! { struct StringBorrower<'a> { optional_str: Option<&'a str>, - optional_string: &Option, - optional_dumb_int: &Option<&i32>, + optional_string: Option<&String>, + optional_dumb_int: Option<&i32>, } };