Run the examples in the new format

ci/scripts/rust_example.sh

@@ -25,12 +25,26 @@ export CARGO_PROFILE_CI_STRIP=true
 cd datafusion-examples/examples/
 cargo build --profile ci --examples
 
-files=$(ls .)
-for filename in $files
-do
-  example_name=`basename $filename ".rs"`
-  # Skip tests that rely on external storage and flight
-  if [ ! -d $filename ]; then
-    cargo run --profile ci --example $example_name
-  fi
+SKIP_LIST=("external_dependency" "flight" "ffi")
+
+skip_example() {
+    local name="$1"
+    for skip in "${SKIP_LIST[@]}"; do
+        if [ "$name" = "$skip" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+for dir in */; do
+    example_name=$(basename "$dir")
+
+    if skip_example "$example_name"; then
+        echo "Skipping $example_name"
+        continue
+    fi
+
+    echo "Running example group: $example_name"
+    cargo run --profile ci --example "$example_name" -- all
 done

datafusion-examples/examples/builtin_functions/main.rs

@@ -21,10 +21,11 @@
 //!
 //! ## Usage
 //! ```bash
-//! cargo run --example builtin_functions -- [date_time|function_factory|regexp]
+//! cargo run --example builtin_functions -- [all|date_time|function_factory|regexp]
 //! ```
 //!
 //! Each subcommand runs a corresponding example:
+//! - `all` — run all examples included in this module
 //! - `date_time` — examples of date-time related functions and queries
 //! - `function_factory` — register `CREATE FUNCTION` handler to implement SQL macros
 //! - `regexp` — examples of using regular expression functions
@@ -38,6 +39,7 @@ use std::str::FromStr;
 use datafusion::error::{DataFusionError, Result};
 
 enum ExampleKind {
+    All,
     DateTime,
     FunctionFactory,
     Regexp,
@@ -46,6 +48,7 @@ enum ExampleKind {
 impl AsRef<str> for ExampleKind {
     fn as_ref(&self) -> &str {
         match self {
+            Self::All => "all",
             Self::DateTime => "date_time",
             Self::FunctionFactory => "function_factory",
             Self::Regexp => "regexp",
@@ -58,6 +61,7 @@ impl FromStr for ExampleKind {
 
     fn from_str(s: &str) -> Result<Self> {
         match s {
+            "all" => Ok(Self::All),
             "date_time" => Ok(Self::DateTime),
             "function_factory" => Ok(Self::FunctionFactory),
             "regexp" => Ok(Self::Regexp),
@@ -67,12 +71,38 @@ impl FromStr for ExampleKind {
 }
 
 impl ExampleKind {
-    const ALL: [Self; 3] = [Self::DateTime, Self::FunctionFactory, Self::Regexp];
+    const ALL_VARIANTS: [Self; 4] = [
+        Self::All,
+        Self::DateTime,
+        Self::FunctionFactory,
+        Self::Regexp,
+    ];
+
+    const RUNNABLE_VARIANTS: [Self; 3] =
+        [Self::DateTime, Self::FunctionFactory, Self::Regexp];
 
     const EXAMPLE_NAME: &str = "builtin_functions";
 
     fn variants() -> Vec<&'static str> {
-        Self::ALL.iter().map(|x| x.as_ref()).collect()
+        Self::ALL_VARIANTS
+            .iter()
+            .map(|example| example.as_ref())
+            .collect()
+    }
+
+    async fn run(&self) -> Result<()> {
+        match self {
+            ExampleKind::All => {
+                for example in ExampleKind::RUNNABLE_VARIANTS {
+                    println!("Running example: {}", example.as_ref());
+                    Box::pin(example.run()).await?;
+                }
+            }
+            ExampleKind::DateTime => date_time::date_time().await?,
+            ExampleKind::FunctionFactory => function_factory::function_factory().await?,
+            ExampleKind::Regexp => regexp::regexp().await?,
+        }
+        Ok(())
     }
 }
 
@@ -89,11 +119,6 @@ async fn main() -> Result<()> {
         DataFusionError::Execution("Missing argument".to_string())
     })?;
 
-    match arg.parse::<ExampleKind>()? {
-        ExampleKind::DateTime => date_time::date_time().await?,
-        ExampleKind::FunctionFactory => function_factory::function_factory().await?,
-        ExampleKind::Regexp => regexp::regexp().await?,
-    }
-
-    Ok(())
+    let example = arg.parse::<ExampleKind>()?;
+    example.run().await
 }

datafusion-examples/examples/builtin_functions/regexp.rs

@@ -18,9 +18,12 @@
 
 //! See `main.rs` for how to run it.
 
+use std::{fs::File, io::Write};
+
 use datafusion::common::{assert_batches_eq, assert_contains};
 use datafusion::error::Result;
 use datafusion::prelude::*;
+use tempfile::tempdir;
 
 /// This example demonstrates how to use the regexp_* functions
 ///
@@ -32,12 +35,30 @@ use datafusion::prelude::*;
 /// https://docs.rs/regex/latest/regex/#grouping-and-flags
 pub async fn regexp() -> Result<()> {
     let ctx = SessionContext::new();
-    ctx.register_csv(
-        "examples",
-        "datafusion/physical-expr/tests/data/regex.csv",
-        CsvReadOptions::new(),
-    )
-    .await?;
+    // content from file 'datafusion/physical-expr/tests/data/regex.csv'
+    let csv_data = r#"values,patterns,replacement,flags
+abc,^(a),bb\1bb,i
+ABC,^(A).*,B,i
+aBc,(b|d),e,i
+AbC,(B|D),e,
+aBC,^(b|c),d,
+4000,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
+4010,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
+Düsseldorf,[\p{Letter}-]+,München,
+Москва,[\p{L}-]+,Moscow,
+Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
+اليوم,^\p{Arabic}+$,Today,"#;
+    let dir = tempdir()?;
+    let file_path = dir.path().join("regex.csv");
+    {
+        let mut file = File::create(&file_path)?;
+        // write CSV data
+        file.write_all(csv_data.as_bytes())?;
+    } // scope closes the file
+    let file_path = file_path.to_str().unwrap();
+
+    ctx.register_csv("examples", file_path, CsvReadOptions::new())
+        .await?;
 
     //
     //

datafusion-examples/examples/custom_data_source/custom_file_casts.rs

@@ -32,7 +32,7 @@ use datafusion::datasource::listing::{
 use datafusion::execution::context::SessionContext;
 use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::parquet::arrow::ArrowWriter;
-use datafusion::physical_expr::expressions::CastExpr;
+use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};
 use datafusion::physical_expr::PhysicalExpr;
 use datafusion::prelude::SessionConfig;
 use datafusion_physical_expr_adapter::{
@@ -192,6 +192,19 @@ impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {
                     );
                 }
             }
+            if let Some(cast) = expr.as_any().downcast_ref::<CastColumnExpr>() {
+                let input_data_type =
+                    cast.expr().data_type(&self.physical_file_schema)?;
+                let output_data_type = cast.data_type(&self.physical_file_schema)?;
+                if !CastExpr::check_bigger_cast(
+                    cast.target_field().data_type(),
+                    &input_data_type,
+                ) {
+                    return not_impl_err!(
+                        "Unsupported CAST from {input_data_type} to {output_data_type}"
+                    );
+                }
+            }
             Ok(Transformed::no(expr))
         })
         .data()

datafusion-examples/examples/custom_data_source/main.rs

@@ -21,10 +21,11 @@
 //!
 //! ## Usage
 //! ```bash
-//! cargo run --example custom_data_source -- [csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]
+//! cargo run --example custom_data_source -- [all|csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]
 //! ```
 //!
 //! Each subcommand runs a corresponding example:
+//! - `all` — run all examples included in this module
 //! - `csv_json_opener` — use low level FileOpener APIs to read CSV/JSON into Arrow RecordBatches
 //! - `csv_sql_streaming` — build and run a streaming query plan from a SQL statement against a local CSV file
 //! - `custom_datasource` — run queries against a custom datasource (TableProvider)
@@ -46,25 +47,27 @@ use std::str::FromStr;
 use datafusion::error::{DataFusionError, Result};
 
 enum ExampleKind {
+    All,
     CsvJsonOpener,
     CsvSqlStreaming,
     CustomDatasource,
     CustomFileCasts,
     CustomFileFormat,
     DefaultColumnValues,
-    FileFtreamProvider,
+    FileStreamProvider,
 }
 
 impl AsRef<str> for ExampleKind {
     fn as_ref(&self) -> &str {
         match self {
+            Self::All => "all",
             Self::CsvJsonOpener => "csv_json_opener",
             Self::CsvSqlStreaming => "csv_sql_streaming",
             Self::CustomDatasource => "custom_datasource",
             Self::CustomFileCasts => "custom_file_casts",
             Self::CustomFileFormat => "custom_file_format",
             Self::DefaultColumnValues => "default_column_values",
-            Self::FileFtreamProvider => "file_stream_provider",
+            Self::FileStreamProvider => "file_stream_provider",
         }
     }
 }
@@ -74,33 +77,79 @@ impl FromStr for ExampleKind {
 
     fn from_str(s: &str) -> Result<Self> {
         match s {
+            "all" => Ok(Self::All),
             "csv_json_opener" => Ok(Self::CsvJsonOpener),
             "csv_sql_streaming" => Ok(Self::CsvSqlStreaming),
             "custom_datasource" => Ok(Self::CustomDatasource),
             "custom_file_casts" => Ok(Self::CustomFileCasts),
             "custom_file_format" => Ok(Self::CustomFileFormat),
             "default_column_values" => Ok(Self::DefaultColumnValues),
-            "file_stream_provider" => Ok(Self::FileFtreamProvider),
+            "file_stream_provider" => Ok(Self::FileStreamProvider),
             _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),
         }
     }
 }
 
 impl ExampleKind {
-    const ALL: [Self; 7] = [
+    const ALL_VARIANTS: [Self; 8] = [
+        Self::All,
         Self::CsvJsonOpener,
         Self::CsvSqlStreaming,
         Self::CustomDatasource,
         Self::CustomFileCasts,
         Self::CustomFileFormat,
         Self::DefaultColumnValues,
-        Self::FileFtreamProvider,
+        Self::FileStreamProvider,
+    ];
+
+    const RUNNABLE_VARIANTS: [Self; 7] = [
+        Self::CsvJsonOpener,
+        Self::CsvSqlStreaming,
+        Self::CustomDatasource,
+        Self::CustomFileCasts,
+        Self::CustomFileFormat,
+        Self::DefaultColumnValues,
+        Self::FileStreamProvider,
     ];
 
     const EXAMPLE_NAME: &str = "custom_data_source";
 
     fn variants() -> Vec<&'static str> {
-        Self::ALL.iter().map(|x| x.as_ref()).collect()
+        Self::ALL_VARIANTS
+            .iter()
+            .map(|example| example.as_ref())
+            .collect()
+    }
+
+    async fn run(&self) -> Result<()> {
+        match self {
+            ExampleKind::All => {
+                for example in ExampleKind::RUNNABLE_VARIANTS {
+                    println!("Running example: {}", example.as_ref());
+                    Box::pin(example.run()).await?;
+                }
+            }
+            ExampleKind::CsvJsonOpener => csv_json_opener::csv_json_opener().await?,
+            ExampleKind::CsvSqlStreaming => {
+                csv_sql_streaming::csv_sql_streaming().await?
+            }
+            ExampleKind::CustomDatasource => {
+                custom_datasource::custom_datasource().await?
+            }
+            ExampleKind::CustomFileCasts => {
+                custom_file_casts::custom_file_casts().await?
+            }
+            ExampleKind::CustomFileFormat => {
+                custom_file_format::custom_file_format().await?
+            }
+            ExampleKind::DefaultColumnValues => {
+                default_column_values::default_column_values().await?
+            }
+            ExampleKind::FileStreamProvider => {
+                file_stream_provider::file_stream_provider().await?
+            }
+        }
+        Ok(())
     }
 }
 
@@ -117,19 +166,6 @@ async fn main() -> Result<()> {
         DataFusionError::Execution("Missing argument".to_string())
     })?;
 
-    match arg.parse::<ExampleKind>()? {
-        ExampleKind::CsvJsonOpener => csv_json_opener::csv_json_opener().await?,
-        ExampleKind::CsvSqlStreaming => csv_sql_streaming::csv_sql_streaming().await?,
-        ExampleKind::CustomDatasource => custom_datasource::custom_datasource().await?,
-        ExampleKind::CustomFileCasts => custom_file_casts::custom_file_casts().await?,
-        ExampleKind::CustomFileFormat => custom_file_format::custom_file_format().await?,
-        ExampleKind::DefaultColumnValues => {
-            default_column_values::default_column_values().await?
-        }
-        ExampleKind::FileFtreamProvider => {
-            file_stream_provider::file_stream_provider().await?
-        }
-    }
-
-    Ok(())
+    let example = arg.parse::<ExampleKind>()?;
+    example.run().await
 }