refactor for Onda Format v0.5.0

.travis.yml

@@ -8,8 +8,8 @@ addons:
     packages:
     - flac
 julia:
-  - 1.0
-  - 1.3
+  - 1.5
+  - nightly
 if: branch = master OR tag IS present OR type = pull_request
 notifications:
   email: false
@@ -18,12 +18,13 @@ notifications:
 #  - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
 #  - julia -e 'Pkg.clone(pwd()); Pkg.build("Onda"); Pkg.test("Onda"; coverage=true)'
 after_success:
-  # push coverage results to Coveralls
   - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'
 jobs:
+  allow_failures:
+    - julia: nightly
   include:
     - stage: "Documentation"
-      julia: 1.0
+      julia: 1.5
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));

Project.toml

@@ -1,24 +1,33 @@
 name = "Onda"
 uuid = "e853f5be-6863-11e9-128d-476edb89bfb5"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.11.0"
+version = "0.12.0"
 
 [deps]
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
 CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 MsgPack = "99f44e22-a591-53d1-9472-aa23ef4bd671"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+TimeSpans = "bb34ddd2-327f-4c4a-bfb0-c98fc494ece1"
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
+Arrow = "1.2.4"
 CodecZstd = "0.6, 0.7"
+ConstructionBase = "1.0"
 MsgPack = "1.1"
+Tables = "1.2"
+TimeSpans = "0.2.2"
 TranscodingStreams = "0.9"
-julia = "1.0"
+julia = "1.5"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 
 [targets]
-test = ["Test"]
+test = ["Test", "DataFrames"]

docs/src/index.md

@@ -1,118 +1,94 @@
 # API Documentation
 
-Below is the documentation for all functions exported by Onda.jl. For general information regarding the Onda format, please see [beacon-biosignals/OndaFormat](https://github.com/beacon-biosignals/OndaFormat).
+Below is the documentation for Onda.jl; for general information regarding the Onda Format itself, please see [beacon-biosignals/OndaFormat](https://github.com/beacon-biosignals/OndaFormat).
 
 ```@meta
 CurrentModule = Onda
 ```
 
-Note that Onda.jl's API follows a specific philosophy with respect to property access: users are generally expected to access fields via Julia's `object.fieldname` syntax, but should only *mutate* objects via the exposed API methods documented below.
+## Support For Generic Path-Like Types
 
-## `Dataset` API
+Onda.jl attempts to be as agnostic as possible with respect to the storage system
+that sample data, Arrow files, etc. are read from/written to. As such, any path-like
+argument accepted by an Onda.jl API function should generically "work" as long
+as the argument's type supports:
+
+- `Base.read(path)::Vector{UInt8}` (return the bytes stored at `path`)
+- `Base.write(path, bytes::Vector{UInt8})` (write `bytes` to the location specified by `path`)
+
+For backends which support direct byte range access (e.g. S3), `Onda.read_byte_range` may
+be overloaded for the backend's corresponding path type to enable further optimizations:
 
 ```@docs
-Dataset
-load
-load_encoded
-save
-create_recording!
-store!
-delete!
-Onda.validate_on_construction
+Onda.read_byte_range
+```
+
+## `*.onda.annotations.arrow`
+
+```@docs
+Annotation
+read_annotations
+write_annotations
+merge_overlapping_annotations
 ```
 
-## Onda Format Metadata
+## `*.onda.signals.arrow`
 
 ```@docs
 Signal
-validate_signal
-signal_from_template
-span
+SamplesInfo
+validate
+read_signals
+write_signals
+channel
+channel_count
+sample_count
 sizeof_samples
-Annotation
-Recording
-set_span!
-annotate!
 ```
 
 ## `Samples`
 
 ```@docs
 Samples
 ==(::Samples, ::Samples)
-validate_samples
 channel
 channel_count
 sample_count
 encode
 encode!
 decode
 decode!
+load
+store
 ```
 
-## `AbstractTimeSpan`
-
-```@docs
-AbstractTimeSpan
-TimeSpan
-contains
-overlaps
-shortest_timespan_containing
-duration
-time_from_index
-index_from_time
-```
-
-## Paths API
-
-Onda's Paths API directly underlies its Dataset API, providing an abstraction
-layer that can be overloaded to support new storage backends for sample data and
-recording metadata. This API's fallback implementation supports any path-like
-type `P` that supports:
-
-- `Base.read(::P)`
-- `Base.write(::P, bytes::Vector{UInt8})`
-- `Base.rm(::P; force, recursive)`
-- `Base.joinpath(::P, ::AbstractString...)`
-- `Base.mkpath(::P)` (note: this is allowed to be a no-op for storage backends which have no notion of intermediate directories, e.g. object storage systems)
-- `Base.dirname(::P)`
-- `Onda.read_byte_range` (see signatures documented below)
-
-```@docs
-read_recordings_file
-write_recordings_file
-samples_path
-read_samples
-write_samples
-read_byte_range
-```
-
-## Serialization API
+## LPCM (De)serialization API
 
-Onda's Serialization API underlies its Paths API, providing a storage-agnostic
-abstraction layer that can be overloaded to support new file/byte formats for
-(de)serializing LPCM-encodeable sample data. This API also facilitates low-level
-streaming sample data (de)serialization and Onda metadata (de)serialization.
+Onda.jl's LPCM (De)serialization API facilitates low-level streaming sample
+data (de)serialization and provides a storage-agnostic abstraction layer
+that can be overloaded to support new file/byte formats for (de)serializing
+LPCM-encodeable sample data.
 
 ```@docs
-deserialize_recordings_msgpack_zst
-serialize_recordings_msgpack_zst
 AbstractLPCMFormat
 AbstractLPCMStream
-deserializing_lpcm_stream
-serializing_lpcm_stream
-finalize_lpcm_stream
-Onda.format_constructor_for_file_extension
+LPCMFormat
+LPCMZstFormat
 format
 deserialize_lpcm
-deserialize_lpcm_callback
 serialize_lpcm
-LPCM
-LPCMZst
+deserialize_lpcm_callback
+deserializing_lpcm_stream
+serializing_lpcm_stream
+finalize_lpcm_stream
+Onda.register_lpcm_format!
+Onda.file_format_string
 ```
 
-## Upgrading Older Datasets to Newer Datasets
+## Utilities
 
 ```@docs
-Onda.upgrade_onda_format_from_v0_2_to_v0_3!
+Onda.gather
+Onda.validate_on_construction
+Onda.upgrade_onda_dataset_to_v0_5!
 ```

examples/flac.jl

@@ -5,63 +5,64 @@
 using Onda, Test, Random, Dates
 
 #####
-##### FLAC
+##### FLACFormat
 #####
 
 """
-     FLAC(lpcm::LPCM; sample_rate, level=5)
-     FLAC(signal::Signal; level=5)
+     FLACFormat(lpcm::LPCMFormat; sample_rate, level=5)
+     FLACFormat(info::SamplesInfo; level=5)
 
-Return a `FLAC<:AbstractLPCMFormat` instance that represents the
-FLAC format assumed for sample data files with the ".flac" extension.
+Return a `FLACFormat<:AbstractLPCMFormat` instance that represents the
+FLAC format corresponding to signals whose `file_format` field is `"flac"`.
 
 The `sample_rate` keyword argument corresponds to `flac`'s `--sample-rate` flag,
 while `level` corresponds to `flac`'s `--compression-level` flag.
 
-Note that FLAC is only applicable for signals where `1 <= signal.channel_count <= 8`
-and `sizeof(signal.sample_type) in (1, 2)`. The corresponding `signal.file_options`
-value may be either `nothing` or `Dict(:level => i)` where `0 <= i <= 8`.
+Note that FLAC is only applicable for `info` where `1 <= channel_count(info) <= 8`
+and `sizeof(info.sample_type) in (1, 2)`.
 
 See https://xiph.org/flac/ for details about FLAC.
 
 See also: [`Zstd`](@ref)
 """
-struct FLAC{S} <: Onda.AbstractLPCMFormat
-    lpcm::LPCM{S}
+struct FLACFormat{S} <: Onda.AbstractLPCMFormat
+    lpcm::LPCMFormat{S}
     sample_rate::Int
     level::Int
-    function FLAC(lpcm::LPCM{S}; sample_rate, level=5) where {S}
+    function FLACFormat(lpcm::LPCMFormat{S}; sample_rate, level=5) where {S}
         sizeof(S) in (1, 2) || throw(ArgumentError("bit depth must be 8 or 16"))
         1 <= lpcm.channel_count <= 8 || throw(ArgumentError("channel count must be between 1 and 8"))
         return new{S}(lpcm, sample_rate, level)
     end
 end
 
-FLAC(signal::Signal; kwargs...) = FLAC(LPCM(signal); sample_rate=signal.sample_rate,
-                                       kwargs...)
+FLACFormat(info::SamplesInfo; kwargs...) = FLACFormat(LPCMFormat(info); sample_rate=info.sample_rate,
+                                                      kwargs...)
 
-Onda.format_constructor_for_file_extension(::Val{:flac}) = FLAC
+Onda.register_lpcm_format!(file_format -> file_format == "flac" ? FLACFormat : nothing)
 
-function flac_raw_specification_flags(serializer::FLAC{S}) where {S}
-    return (level="--compression-level-$(serializer.level)",
+file_format_string(::FLACFormat) = "flac"
+
+function flac_raw_specification_flags(format::FLACFormat{S}) where {S}
+    return (level="--compression-level-$(format.level)",
             endian="--endian=little",
-            channels="--channels=$(serializer.lpcm.channel_count)",
+            channels="--channels=$(format.lpcm.channel_count)",
             bps="--bps=$(sizeof(S) * 8)",
-            sample_rate="--sample-rate=$(serializer.sample_rate)",
+            sample_rate="--sample-rate=$(format.sample_rate)",
             is_signed=string("--sign=", S <: Signed ? "signed" : "unsigned"))
 end
 
 struct FLACStream{L<:Onda.LPCMStream} <: AbstractLPCMStream
     stream::L
 end
 
-function Onda.deserializing_lpcm_stream(format::FLAC, io)
+function Onda.deserializing_lpcm_stream(format::FLACFormat, io)
     flags = flac_raw_specification_flags(format)
     cmd = open(`flac - --totally-silent -d --force-raw-format $(flags.endian) $(flags.is_signed)`, io)
     return FLACStream(Onda.LPCMStream(format.lpcm, cmd))
 end
 
-function Onda.serializing_lpcm_stream(format::FLAC, io)
+function Onda.serializing_lpcm_stream(format::FLACFormat, io)
     flags = flac_raw_specification_flags(format)
     cmd = open(`flac --totally-silent $(flags) -`, io; write=true)
     return FLACStream(Onda.LPCMStream(format.lpcm, cmd))
@@ -77,14 +78,14 @@ Onda.deserialize_lpcm(stream::FLACStream, args...) = deserialize_lpcm(stream.str
 
 Onda.serialize_lpcm(stream::FLACStream, args...) = serialize_lpcm(stream.stream, args...)
 
-function Onda.deserialize_lpcm(format::FLAC, bytes, args...)
+function Onda.deserialize_lpcm(format::FLACFormat, bytes, args...)
     stream = deserializing_lpcm_stream(format, IOBuffer(bytes))
     results = deserialize_lpcm(stream, args...)
     finalize_lpcm_stream(stream)
     return results
 end
 
-function Onda.serialize_lpcm(format::FLAC, samples::AbstractMatrix)
+function Onda.serialize_lpcm(format::FLACFormat, samples::AbstractMatrix)
     io = IOBuffer()
     stream = serializing_lpcm_stream(format, io)
     serialize_lpcm(stream, samples)
@@ -96,28 +97,37 @@ end
 ##### tests
 #####
 
+saws(info, duration) = [(j + i) % 100 * info.sample_resolution_in_unit for
+                        i in 1:channel_count(info), j in 1:sample_count(info, duration)]
+
 if VERSION >= v"1.1.0"
     @testset "FLAC example" begin
-        signal = Signal([:a, :b, :c], Nanosecond(0), Nanosecond(0), :unit, 0.25, -0.5, Int16, 50, :flac, Dict(:level => 2))
-        samples = encode(Samples(signal, false, rand(MersenneTwister(1), 3, Int(50 * 10))))
-        signal_format = format(signal)
-
-        bytes = serialize_lpcm(signal_format, samples.data)
-        @test deserialize_lpcm(signal_format, bytes) == samples.data
-        @test deserialize_lpcm(signal_format, bytes, 99) == view(samples.data, :, 100:size(samples.data, 2))
-        @test deserialize_lpcm(signal_format, bytes, 99, 201) == view(samples.data, :, 100:300)
+        info = SamplesInfo(; kind="test", channels=["a", "b", "c"],
+                           sample_unit="unit",
+                           sample_resolution_in_unit=0.25,
+                           sample_offset_in_unit=-0.5,
+                           sample_type=Int16,
+                           sample_rate=50)
+        data = saws(info, Minute(3))
+        samples = encode(Samples(data, info, false))
+        fmt = FLACFormat(info)
+
+        bytes = serialize_lpcm(fmt, samples.data)
+        @test deserialize_lpcm(fmt, bytes) == samples.data
+        @test deserialize_lpcm(fmt, bytes, 99) == view(samples.data, :, 100:size(samples.data, 2))
+        @test deserialize_lpcm(fmt, bytes, 99, 201) == view(samples.data, :, 100:300)
 
         io = IOBuffer()
-        stream = serializing_lpcm_stream(signal_format, io)
+        stream = serializing_lpcm_stream(fmt, io)
         serialize_lpcm(stream, samples.data)
         @test finalize_lpcm_stream(stream)
         seekstart(io)
-        stream = deserializing_lpcm_stream(signal_format, io)
+        stream = deserializing_lpcm_stream(fmt, io)
         @test deserialize_lpcm(stream) == samples.data
         finalize_lpcm_stream(stream) && close(io)
 
         io = IOBuffer(bytes)
-        stream = deserializing_lpcm_stream(signal_format, io)
+        stream = deserializing_lpcm_stream(fmt, io)
         @test deserialize_lpcm(stream, 49, 51) == view(samples.data, :, 50:100)
         @test deserialize_lpcm(stream, 49, 51) == view(samples.data, :, 150:200)
         @test deserialize_lpcm(stream, 9) == view(samples.data, :, 210:size(samples.data, 2))