Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions csv/read.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/cloudquery/plugin-sdk/v4/schema"
)

func (cl *Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error {
func (cl *Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.RecordBatch) error {
arrowSchema := table.ToArrowSchema()
newSchema := convertSchema(arrowSchema)
reader := csv.NewReader(r, newSchema,
Expand All @@ -23,7 +23,7 @@ func (cl *Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- a
if reader.Err() != nil {
return reader.Err()
}
rec := reader.Record()
rec := reader.RecordBatch()
castRec, err := castFromString(rec, arrowSchema)
if err != nil {
return fmt.Errorf("failed to cast extension types: %w", err)
Expand All @@ -34,7 +34,7 @@ func (cl *Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- a
}

// castFromString casts extension columns to string.
func castFromString(rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record, error) {
func castFromString(rec arrow.RecordBatch, arrowSchema *arrow.Schema) (arrow.RecordBatch, error) {
cols := make([]arrow.Array, rec.NumCols())
for c, f := range arrowSchema.Fields() {
col := rec.Column(c)
Expand All @@ -55,5 +55,5 @@ func castFromString(rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record,
}
cols[c] = sb.NewArray()
}
return array.NewRecord(arrowSchema, cols, rec.NumRows()), nil
return array.NewRecordBatch(arrowSchema, cols, rec.NumRows()), nil
}
4 changes: 2 additions & 2 deletions csv/testdata/TestWriteRead-default.csv

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions csv/testdata/TestWriteRead-with_delimiter.csv

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions csv/testdata/TestWriteRead-with_delimiter_headers.csv

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions csv/testdata/TestWriteRead-with_headers.csv

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions csv/write.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (cl *Client) WriteHeader(w io.Writer, t *schema.Table) (types.Handle, error
}, nil
}

func (h *Handle) WriteContent(records []arrow.Record) error {
func (h *Handle) WriteContent(records []arrow.RecordBatch) error {
for _, record := range records {
castRec := h.castToString(record)
if err := h.w.Write(castRec); err != nil {
Expand Down Expand Up @@ -91,7 +91,7 @@ func isTypeSupported(t arrow.DataType) bool {
}

// castToString casts extension columns or unsupported columns to string. It does not release the original record.
func (h *Handle) castToString(rec arrow.Record) arrow.Record {
func (h *Handle) castToString(rec arrow.RecordBatch) arrow.RecordBatch {
cols := make([]arrow.Array, h.schema.NumFields())
for c := 0; c < h.schema.NumFields(); c++ {
col := rec.Column(c)
Expand All @@ -110,7 +110,7 @@ func (h *Handle) castToString(rec arrow.Record) arrow.Record {
}
cols[c] = sb.NewArray()
}
return array.NewRecord(h.schema, cols, rec.NumRows())
return array.NewRecordBatch(h.schema, cols, rec.NumRows())
}

func stripCQExtensionMetadata(md arrow.Metadata) arrow.Metadata {
Expand Down
10 changes: 5 additions & 5 deletions csv/write_read_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestWriteRead(t *testing.T) {
writer := bufio.NewWriter(&b)
reader := bufio.NewReader(&b)

if err := types.WriteAll(cl, writer, table, []arrow.Record{record}); err != nil {
if err := types.WriteAll(cl, writer, table, []arrow.RecordBatch{record}); err != nil {
t.Fatal(err)
}
writer.Flush()
Expand All @@ -67,17 +67,17 @@ func TestWriteRead(t *testing.T) {

byteReader := bytes.NewReader(rawBytes)

ch := make(chan arrow.Record)
ch := make(chan arrow.RecordBatch)
var readErr error
go func() {
readErr = cl.Read(byteReader, table, ch)
close(ch)
}()
received := make([]arrow.Record, 0, tc.outputCount)
received := make([]arrow.RecordBatch, 0, tc.outputCount)
for got := range ch {
received = append(received, got)
}
require.Empty(t, plugin.RecordsDiff(table.ToArrowSchema(), []arrow.Record{record}, received))
require.Empty(t, plugin.RecordsDiff(table.ToArrowSchema(), []arrow.RecordBatch{record}, received))
require.NoError(t, readErr)
require.Equalf(t, tc.outputCount, len(received), "got %d row(s), want %d", len(received), tc.outputCount)
})
Expand All @@ -104,7 +104,7 @@ func BenchmarkWrite(b *testing.B) {
writer := bufio.NewWriter(&buf)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := types.WriteAll(cl, writer, table, []arrow.Record{record}); err != nil {
if err := types.WriteAll(cl, writer, table, []arrow.RecordBatch{record}); err != nil {
b.Fatal(err)
}

Expand Down
28 changes: 14 additions & 14 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ module github.com/cloudquery/filetypes/v4
go 1.25.1

require (
github.com/apache/arrow-go/v18 v18.4.0
github.com/apache/arrow-go/v18 v18.5.0
github.com/bradleyjkemp/cupaloy/v2 v2.8.0
github.com/cloudquery/codegen v0.3.33
github.com/cloudquery/plugin-sdk/v4 v4.93.0
github.com/cloudquery/plugin-sdk/v4 v4.93.1
github.com/goccy/go-json v0.10.5
github.com/invopop/jsonschema v0.13.0
github.com/stretchr/testify v1.11.1
Expand All @@ -20,16 +20,16 @@ require (
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudquery/cloudquery-api-go v1.14.6 // indirect
github.com/cloudquery/cloudquery-api-go v1.14.7 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/flatbuffers v25.9.23+incompatible // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
Expand All @@ -43,18 +43,18 @@ require (
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/thoas/go-funk v0.9.3 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect
golang.org/x/mod v0.31.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/tools v0.38.0 // indirect
golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect
golang.org/x/text v0.32.0 // indirect
golang.org/x/tools v0.40.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/grpc v1.77.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

Expand Down
56 changes: 28 additions & 28 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/apache/arrow-go/v18 v18.4.0 h1:/RvkGqH517iY8bZKc4FD5/kkdwXJGjxf28JIXbJ/oB0=
github.com/apache/arrow-go/v18 v18.4.0/go.mod h1:Aawvwhj8x2jURIzD9Moy72cF0FyJXOpkYpdmGRHcw14=
github.com/apache/arrow-go/v18 v18.5.0 h1:rmhKjVA+MKVnQIMi/qnM0OxeY4tmHlN3/Pvu+Itmd6s=
github.com/apache/arrow-go/v18 v18.5.0/go.mod h1:F1/wPb3bUy6ZdP4kEPWC7GUZm+yDmxXFERK6uDSkhr8=
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
Expand All @@ -16,14 +16,14 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudquery/cloudquery-api-go v1.14.6 h1:lY0ndk5XAe84EronTCg8AnB40FwHCbAYpM4QEPzEb9E=
github.com/cloudquery/cloudquery-api-go v1.14.6/go.mod h1:d+I8E+z3vmvTvCNXZ5YNhxV9InY/i1siXa0vCYq+ABk=
github.com/cloudquery/cloudquery-api-go v1.14.7 h1:j8XkwY7YMtZnXi655BTFLqadCs9PJtISkw9Au2+/8QY=
github.com/cloudquery/cloudquery-api-go v1.14.7/go.mod h1:d+I8E+z3vmvTvCNXZ5YNhxV9InY/i1siXa0vCYq+ABk=
github.com/cloudquery/codegen v0.3.33 h1:EsQjaiupoTgUvwYNmeHy2Wj3XWrpXXojnGZmnV9GpLM=
github.com/cloudquery/codegen v0.3.33/go.mod h1:abfJ0HDW4Elh+ppwbKrCuAsXfZUmaWyY+4SgUyfQndg=
github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66 h1:OZLPSIBYEfvkAUeOeM8CwTgVQy5zhayI99ishCrsFV0=
github.com/cloudquery/jsonschema v0.0.0-20240220124159-92878faa2a66/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
github.com/cloudquery/plugin-sdk/v4 v4.93.0 h1:JJSH8q8d0bMdm7aU2zBM4t2lBim5kyAs2Nr1c0ZRN7I=
github.com/cloudquery/plugin-sdk/v4 v4.93.0/go.mod h1:aTEk8USXw57jW8dnsEwIZgF0T2e3SsifAqjUK1dIZZQ=
github.com/cloudquery/plugin-sdk/v4 v4.93.1 h1:doZ1CA6LYgatqsJvfcOE7PNOaIbvktsWi4vEGyNnqQw=
github.com/cloudquery/plugin-sdk/v4 v4.93.1/go.mod h1:1gs7aC8I7QAHu5FPffFUSHC/TjFv6V8w1gRLvd/U93g=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -44,8 +44,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU=
github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
Expand All @@ -60,10 +60,10 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
Expand Down Expand Up @@ -127,25 +127,25 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 h1:MDfG8Cvcqlt9XXrmEiD4epKn7VJHZO84hejP9Jmp0MM=
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU=
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc h1:bH6xUXay0AIFMElXG2rQ4uiE+7ncwtiOdPfYK1NK2XA=
golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
Expand All @@ -154,8 +154,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
4 changes: 2 additions & 2 deletions json/read.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

const maxJSONSize = 1024 * 1024 * 20

func (*Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error {
func (*Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.RecordBatch) error {
scanner := bufio.NewScanner(r)
scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize)
rb := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema())
Expand All @@ -22,7 +22,7 @@ func (*Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arro
if err != nil {
return err
}
r := rb.NewRecord()
r := rb.NewRecordBatch()
res <- r
}

Expand Down
4 changes: 2 additions & 2 deletions json/testdata/TestWriteRead.jsonl

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions json/write.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (*Handle) WriteFooter() error {
return nil
}

func (h *Handle) WriteContent(records []arrow.Record) error {
func (h *Handle) WriteContent(records []arrow.RecordBatch) error {
for _, r := range records {
err := writeRecord(h.w, r)
if err != nil {
Expand All @@ -36,7 +36,7 @@ func (h *Handle) WriteContent(records []arrow.Record) error {
return nil
}

func writeRecord(w io.Writer, record arrow.Record) error {
func writeRecord(w io.Writer, record arrow.RecordBatch) error {
arr := array.RecordToStructArray(record)
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
Expand Down
10 changes: 5 additions & 5 deletions json/write_read_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestWriteRead(t *testing.T) {
writer := bufio.NewWriter(&b)
reader := bufio.NewReader(&b)

if err := types.WriteAll(cl, writer, table, []arrow.Record{record}); err != nil {
if err := types.WriteAll(cl, writer, table, []arrow.RecordBatch{record}); err != nil {
t.Fatal(err)
}
writer.Flush()
Expand All @@ -54,17 +54,17 @@ func TestWriteRead(t *testing.T) {

byteReader := bytes.NewReader(rawBytes)

ch := make(chan arrow.Record)
ch := make(chan arrow.RecordBatch)
var readErr error
go func() {
readErr = cl.Read(byteReader, table, ch)
close(ch)
}()
received := make([]arrow.Record, 0, 2)
received := make([]arrow.RecordBatch, 0, 2)
for got := range ch {
received = append(received, got)
}
require.Empty(t, plugin.RecordsDiff(table.ToArrowSchema(), []arrow.Record{record}, received))
require.Empty(t, plugin.RecordsDiff(table.ToArrowSchema(), []arrow.RecordBatch{record}, received))
require.NoError(t, readErr)
require.Equalf(t, 2, len(received), "got %d row(s), want %d", len(received), 2)
}
Expand All @@ -89,7 +89,7 @@ func BenchmarkWrite(b *testing.B) {
writer := bufio.NewWriter(&buf)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := types.WriteAll(cl, writer, table, []arrow.Record{record}); err != nil {
if err := types.WriteAll(cl, writer, table, []arrow.RecordBatch{record}); err != nil {
b.Fatal(err)
}
err = writer.Flush()
Expand Down
Loading