diff --git a/http/get_simple/julia/Project.toml b/http/get_simple/julia/Project.toml new file mode 100644 index 0000000..0929efe --- /dev/null +++ b/http/get_simple/julia/Project.toml @@ -0,0 +1,5 @@ +[deps] +Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" diff --git a/http/get_simple/julia/client/README.md b/http/get_simple/julia/client/README.md new file mode 100644 index 0000000..3dbed30 --- /dev/null +++ b/http/get_simple/julia/client/README.md @@ -0,0 +1,32 @@ + + +# HTTP GET Arrow Data: Simple Julia Client Example + +This directory contains a minimal example of an HTTP client implemented in Julia. The client: +1. Sends an HTTP GET request to a server. +2. Receives an HTTP 200 response from the server, with the response body containing an Arrow IPC stream of record batches. +3. Adds the record batches to a list as they are received. + +To run this example, first start one of the server examples in the parent directory, then: + +```sh +julia --project=.. -e "using Pkg; Pkg.instantiate()" +julia --project=.. client.jl +``` diff --git a/http/get_simple/julia/client/client.jl b/http/get_simple/julia/client/client.jl new file mode 100644 index 0000000..1aa2073 --- /dev/null +++ b/http/get_simple/julia/client/client.jl @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +using Arrow, HTTP + +function get_batches() + res = HTTP.get("http://localhost:8008") + buffer = res.body + stream = Arrow.Stream(res.body) + batches = collect(stream) + + println("$(length(buffer)) bytes received") + println("$(length(batches)) record batches received") + + return batches +end + +execution_time = @elapsed get_batches() +println("$(execution_time) seconds elapsed") diff --git a/http/get_simple/julia/server/README.md b/http/get_simple/julia/server/README.md new file mode 100644 index 0000000..19abc3a --- /dev/null +++ b/http/get_simple/julia/server/README.md @@ -0,0 +1,32 @@ + + +# HTTP GET Arrow Data: Simple Julia Server Example + +This directory contains a minimal example of an HTTP server implemented in Julia. The server: +1. Creates a list of record batches and populates it with synthesized data. +2. Listens for HTTP GET requests from clients. +3. Upon receiving a request, sends an HTTP 200 response with the body containing an Arrow IPC stream of record batches. + +To run this example: + +```sh +julia --project=.. -e "using Pkg; Pkg.instantiate()" +julia --project=.. server.jl +``` diff --git a/http/get_simple/julia/server/server.jl b/http/get_simple/julia/server/server.jl new file mode 100644 index 0000000..8f56524 --- /dev/null +++ b/http/get_simple/julia/server/server.jl @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +using Arrow, HTTP, Random, Tables + +function randint_nullable(n::Integer) + v = Vector{Union{Missing, Int}}(undef, n) + rand!(v, Int) + return v +end + +function get_stream(::HTTP.Request) + total_records = 100_000_000 + batch_len = 4096 + stream = Tables.partitioner(Iterators.partition(1:total_records, batch_len)) do indices + nrows = length(indices) + return ( + a = randint_nullable(nrows), + b = randint_nullable(nrows), + c = randint_nullable(nrows), + d = randint_nullable(nrows) + ) + end + buffer = IOBuffer() + Arrow.write(buffer, stream) + return HTTP.Response(200, take!(buffer)) +end + +println("Serving on localhost:8008...") +server = HTTP.serve(get_stream, "127.0.0.1", 8008)