diff --git a/Cargo.lock b/Cargo.lock index dfe41e936c..03c3bb7ca9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4429,7 +4429,7 @@ dependencies = [ [[package]] name = "iggy" -version = "0.8.1-edge.7" +version = "0.8.2-edge.1" dependencies = [ "async-broadcast", "async-dropper", @@ -4461,7 +4461,7 @@ dependencies = [ [[package]] name = "iggy-bench" -version = "0.3.1-edge.2" +version = "0.3.2-edge.1" dependencies = [ "async-trait", "bench-report", @@ -4516,7 +4516,7 @@ dependencies = [ [[package]] name = "iggy-cli" -version = "0.10.1-edge.1" +version = "0.10.2-edge.1" dependencies = [ "ahash 0.8.12", "anyhow", @@ -4536,7 +4536,7 @@ dependencies = [ [[package]] name = "iggy-connectors" -version = "0.2.1-edge.6" +version = "0.2.2-edge.1" dependencies = [ "async-trait", "axum", @@ -4588,7 +4588,7 @@ dependencies = [ [[package]] name = "iggy-mcp" -version = "0.2.1-edge.5" +version = "0.2.2-edge.1" dependencies = [ "axum", "axum-server", @@ -4620,7 +4620,7 @@ dependencies = [ [[package]] name = "iggy_binary_protocol" -version = "0.8.1-edge.3" +version = "0.8.2-edge.1" dependencies = [ "anyhow", "async-broadcast", @@ -4641,7 +4641,7 @@ dependencies = [ [[package]] name = "iggy_common" -version = "0.8.1-edge.2" +version = "0.8.2-edge.1" dependencies = [ "aes-gcm", "ahash 0.8.12", @@ -4684,7 +4684,7 @@ dependencies = [ [[package]] name = "iggy_connector_elasticsearch_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "base64 0.22.1", @@ -4702,7 +4702,7 @@ dependencies = [ [[package]] name = "iggy_connector_elasticsearch_source" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "chrono", @@ -4720,7 +4720,7 @@ dependencies = [ [[package]] name = "iggy_connector_iceberg_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "arrow-json", "async-trait", @@ -4739,7 +4739,7 @@ dependencies = [ [[package]] name = "iggy_connector_postgres_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "chrono", @@ -4758,7 +4758,7 @@ dependencies = [ [[package]] name = "iggy_connector_postgres_source" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "base64 0.22.1", @@ -4779,7 +4779,7 @@ dependencies = [ [[package]] name = "iggy_connector_quickwit_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "dashmap", @@ -4794,7 +4794,7 @@ dependencies = [ [[package]] name = "iggy_connector_random_source" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "dashmap", @@ -4811,7 +4811,7 @@ dependencies = [ [[package]] name = "iggy_connector_sdk" -version = "0.1.1-edge.3" +version = "0.1.2-edge.1" dependencies = [ "async-trait", "base64 0.22.1", @@ -4839,7 +4839,7 @@ dependencies = [ [[package]] name = "iggy_connector_stdout_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" dependencies = [ "async-trait", "dashmap", @@ -4852,7 +4852,7 @@ dependencies = [ [[package]] name = "iggy_examples" -version = "0.0.5" +version = "0.0.6" dependencies = [ "ahash 0.8.12", "anyhow", @@ -8251,7 +8251,7 @@ dependencies = [ [[package]] name = "server" -version = "0.6.1-edge.6" +version = "0.6.2-edge.1" dependencies = [ "ahash 0.8.12", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index b8c8e4594e..b8ea12417d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,10 +140,10 @@ humantime = "2.3.0" hwlocality = "1.0.0-alpha.11" iceberg = "0.8.0" iceberg-catalog-rest = "0.8.0" -iggy = { path = "core/sdk", version = "0.8.1-edge.6" } -iggy_binary_protocol = { path = "core/binary_protocol", version = "0.8.1-edge.3" } -iggy_common = { path = "core/common", version = "0.8.1-edge.2" } -iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.1.1-edge.3" } +iggy = { path = "core/sdk", version = "0.8.2-edge.1" } +iggy_binary_protocol = { path = "core/binary_protocol", version = "0.8.2-edge.1" } +iggy_common = { path = "core/common", version = "0.8.2-edge.1" } +iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.1.2-edge.1" } integration = { path = "core/integration" } keyring = { version = "3.6.3", features = ["sync-secret-service", "vendored"] } lazy_static = "1.5.0" diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index b5861ebeaa..306f3bdb7e 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -385,24 +385,24 @@ icu_provider: 2.1.1, "Unicode-3.0", ident_case: 1.0.1, "Apache-2.0 OR MIT", idna: 1.1.0, "Apache-2.0 OR MIT", idna_adapter: 1.2.1, "Apache-2.0 OR MIT", -iggy: 0.8.1-edge.7, "Apache-2.0", -iggy-bench: 0.3.1-edge.2, "Apache-2.0", +iggy: 0.8.2-edge.1, "Apache-2.0", +iggy-bench: 0.3.2-edge.1, "Apache-2.0", iggy-bench-dashboard-server: 0.5.1-edge.1, "Apache-2.0", -iggy-cli: 0.10.1-edge.1, "Apache-2.0", -iggy-connectors: 0.2.1-edge.6, "Apache-2.0", -iggy-mcp: 0.2.1-edge.5, "Apache-2.0", -iggy_binary_protocol: 0.8.1-edge.3, "Apache-2.0", -iggy_common: 0.8.1-edge.2, "Apache-2.0", -iggy_connector_elasticsearch_sink: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_elasticsearch_source: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_iceberg_sink: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_postgres_sink: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_postgres_source: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_quickwit_sink: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_random_source: 0.2.0-edge.1, "Apache-2.0", -iggy_connector_sdk: 0.1.1-edge.3, "Apache-2.0", -iggy_connector_stdout_sink: 0.2.0-edge.1, "Apache-2.0", -iggy_examples: 0.0.5, "Apache-2.0", +iggy-cli: 0.10.2-edge.1, "Apache-2.0", +iggy-connectors: 0.2.2-edge.1, "Apache-2.0", +iggy-mcp: 0.2.2-edge.1, "Apache-2.0", +iggy_binary_protocol: 0.8.2-edge.1, "Apache-2.0", +iggy_common: 0.8.2-edge.1, "Apache-2.0", +iggy_connector_elasticsearch_sink: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_elasticsearch_source: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_iceberg_sink: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_postgres_sink: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_postgres_source: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_quickwit_sink: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_random_source: 0.2.1-edge.1, "Apache-2.0", +iggy_connector_sdk: 0.1.2-edge.1, "Apache-2.0", +iggy_connector_stdout_sink: 0.2.1-edge.1, "Apache-2.0", +iggy_examples: 0.0.6, "Apache-2.0", ignore: 0.4.25, "MIT OR Unlicense", impl-more: 0.1.9, "Apache-2.0 OR MIT", implicit-clone: 0.6.0, "Apache-2.0 OR MIT", @@ -718,7 +718,7 @@ serde_with_macros: 3.16.1, "Apache-2.0 OR MIT", serde_yaml_ng: 0.10.0, "MIT", serial_test: 3.3.1, "MIT", serial_test_derive: 3.3.1, "MIT", -server: 0.6.1-edge.6, "Apache-2.0", +server: 0.6.2-edge.1, "Apache-2.0", sha1: 0.10.6, "Apache-2.0 OR MIT", sha2: 0.10.9, "Apache-2.0 OR MIT", sha3: 0.10.8, "Apache-2.0 OR MIT", diff --git a/bdd/go/tests/tcp_test/messages_steps.go b/bdd/go/tests/tcp_test/messages_steps.go index e364f62282..c695b0e24f 100644 --- a/bdd/go/tests/tcp_test/messages_steps.go +++ b/bdd/go/tests/tcp_test/messages_steps.go @@ -28,10 +28,10 @@ import ( "github.com/onsi/gomega" ) -func createDefaultMessageHeaders() map[iggcon.HeaderKey]iggcon.HeaderValue { - return map[iggcon.HeaderKey]iggcon.HeaderValue{ - {Value: createRandomString(4)}: {Kind: iggcon.String, Value: []byte(createRandomString(8))}, - {Value: createRandomString(8)}: {Kind: iggcon.Uint32, Value: []byte{0x01, 0x02, 0x03, 0x04}}, +func createDefaultMessageHeaders() []iggcon.HeaderEntry { + return []iggcon.HeaderEntry{ + {Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(createRandomString(4))}, Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(createRandomString(8))}}, + {Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(createRandomString(8))}, Value: iggcon.HeaderValue{Kind: iggcon.Uint32, Value: []byte{0x01, 0x02, 0x03, 0x04}}}, } } diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml index 8cb193aa70..d7b781f5ee 100644 --- a/core/ai/mcp/Cargo.toml +++ b/core/ai/mcp/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy-mcp" -version = "0.2.1-edge.5" +version = "0.2.2-edge.1" description = "MCP Server for Iggy message streaming platform" edition = "2024" license = "Apache-2.0" diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml index bbfbae0358..ea475e412f 100644 --- a/core/bench/Cargo.toml +++ b/core/bench/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy-bench" -version = "0.3.1-edge.2" +version = "0.3.2-edge.1" edition = "2024" license = "Apache-2.0" repository = "https://github.com/apache/iggy" diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index c015b52576..32e237bc4c 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_binary_protocol" -version = "0.8.1-edge.3" +version = "0.8.2-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/binary_protocol/src/cli/binary_message/poll_messages.rs b/core/binary_protocol/src/cli/binary_message/poll_messages.rs index d4867a4a35..7b6725512c 100644 --- a/core/binary_protocol/src/cli/binary_message/poll_messages.rs +++ b/core/binary_protocol/src/cli/binary_message/poll_messages.rs @@ -88,7 +88,7 @@ impl PollMessagesCmd { match HashMap::::from_bytes(user_headers.clone()) { Ok(headers) => headers .iter() - .map(|(k, v)| (k.clone(), v.kind)) + .map(|(k, v)| (k.clone(), v.kind())) .collect::>(), Err(e) => { tracing::error!("Failed to parse user headers, error: {e}"); @@ -113,7 +113,7 @@ impl PollMessagesCmd { let message_headers = header_key_set .iter() .map(|(key, kind)| { - Cell::new(format!("Header: {}\n{}", key.as_str(), kind)) + Cell::new(format!("Header: {}\n{}", key.to_string_value(), kind)) .set_alignment(CellAlignment::Center) }) .collect::>(); @@ -146,8 +146,8 @@ impl PollMessagesCmd { .as_ref() .map(|h| { h.get(key) - .filter(|v| v.kind == *kind) - .map(|v| v.value_only_to_string()) + .filter(|v| v.kind() == *kind) + .map(|v| v.to_string_value()) .unwrap_or_default() }) .unwrap_or_default() diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index 98fd9b7f10..a7f2b5382c 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy-cli" -version = "0.10.1-edge.1" +version = "0.10.2-edge.1" edition = "2024" authors = ["bartosz.ciesla@gmail.com"] repository = "https://github.com/apache/iggy" diff --git a/core/cli/src/args/message.rs b/core/cli/src/args/message.rs index da5e742e62..9b70df7024 100644 --- a/core/cli/src/args/message.rs +++ b/core/cli/src/args/message.rs @@ -120,15 +120,73 @@ pub(crate) struct SendMessagesArgs { /// Parse Header Key, Kind and Value from the string separated by a ':' fn parse_key_val(s: &str) -> Result<(HeaderKey, HeaderValue), IggyError> { - let lower = s.to_lowercase(); - let parts = lower.split(':').collect::>(); + let parts = s.splitn(3, ':').collect::>(); if parts.len() != 3 { return Err(IggyError::InvalidFormat); } let key = HeaderKey::from_str(parts[0])?; - let value = HeaderValue::from_kind_str_and_value_str(parts[1], parts[2])?; + let kind = HeaderKind::from_str(&parts[1].to_lowercase())?; + let value_str = parts[2]; + + let value = match kind { + HeaderKind::Raw => HeaderValue::try_from(value_str.as_bytes())?, + HeaderKind::String => HeaderValue::try_from(value_str)?, + HeaderKind::Bool => value_str + .parse::() + .map_err(|_| IggyError::InvalidBooleanValue)? + .into(), + HeaderKind::Int8 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Int16 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Int32 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Int64 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Int128 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Uint8 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Uint16 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Uint32 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Uint64 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Uint128 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Float32 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + HeaderKind::Float64 => value_str + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)? + .into(), + }; + Ok((key, value)) } @@ -292,4 +350,23 @@ mod tests { let result = parse_key_val("key:uint8:69.42"); assert!(result.is_err()); } + + #[test] + fn parse_key_val_should_preserve_value_case() { + let expected_value = "HelloWorld"; + let result = parse_key_val(&format!("key:string:{expected_value}")); + assert!(result.is_ok()); + let (_, value) = result.unwrap(); + assert_eq!(value.as_str().unwrap(), expected_value); + } + + #[test] + fn parse_key_val_should_preserve_colons_in_value() { + let expected_value = "http://example.com:8080"; + let result = parse_key_val(&format!("url:string:{expected_value}")); + assert!(result.is_ok()); + let (key, value) = result.unwrap(); + assert_eq!(key, HeaderKey::from_str("url").unwrap()); + assert_eq!(value.as_str().unwrap(), expected_value); + } } diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml index fbcb02cd74..07a82abc7b 100644 --- a/core/common/Cargo.toml +++ b/core/common/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [package] name = "iggy_common" -version = "0.8.1-edge.2" +version = "0.8.2-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/common/src/commands/messages/send_messages.rs b/core/common/src/commands/messages/send_messages.rs index 05a4187aba..c4f2913e30 100644 --- a/core/common/src/commands/messages/send_messages.rs +++ b/core/common/src/commands/messages/send_messages.rs @@ -23,6 +23,7 @@ use crate::PartitioningKind; use crate::Sizeable; use crate::Validatable; use crate::error::IggyError; +use crate::types::message::HeaderEntry; use crate::types::message::partitioning::Partitioning; use crate::{Command, SEND_MESSAGES_CODE}; use crate::{INDEX_SIZE, IggyMessage, IggyMessagesBatch}; @@ -203,7 +204,11 @@ impl Serialize for SendMessages { map.insert("payload", serde_json::to_value(payload_base64).unwrap()); if let Ok(Some(headers)) = msg_view.user_headers_map() { - map.insert("headers", serde_json::to_value(&headers).unwrap()); + let entries: Vec = headers + .into_iter() + .map(|(k, v)| HeaderEntry { key: k, value: v }) + .collect(); + map.insert("user_headers", serde_json::to_value(&entries).unwrap()); } map @@ -305,13 +310,23 @@ impl<'de> Deserialize<'de> for SendMessages { .decode(payload) .map_err(|_| de::Error::custom("Invalid base64 payload"))?; - let headers_map = if let Some(headers) = msg.get("headers") { + let headers_map = if let Some(headers) = msg.get("user_headers") { if headers.is_null() { None } else { - Some(serde_json::from_value(headers.clone()).map_err( - |_| de::Error::custom("Invalid headers format"), - )?) + let entries: Vec = serde_json::from_value( + headers.clone(), + ) + .map_err(|e| { + de::Error::custom(format!( + "Invalid headers format: {e}" + )) + })?; + let mut map = HashMap::new(); + for entry in entries { + map.insert(entry.key, entry.value); + } + Some(map) } } else { None diff --git a/core/common/src/types/message/iggy_message.rs b/core/common/src/types/message/iggy_message.rs index 5998c39751..49fe0b89be 100644 --- a/core/common/src/types/message/iggy_message.rs +++ b/core/common/src/types/message/iggy_message.rs @@ -541,22 +541,29 @@ impl Serialize for IggyMessage { where S: Serializer, { + use super::user_headers::HeaderEntry; use base64::{Engine as _, engine::general_purpose::STANDARD}; use serde::ser::SerializeStruct; - let field_count = 2 + self.user_headers.is_some() as usize; - - let mut state = serializer.serialize_struct("IggyMessage", field_count)?; + let mut state = serializer.serialize_struct("IggyMessage", 3)?; state.serialize_field("header", &self.header)?; let base64_payload = STANDARD.encode(&self.payload); state.serialize_field("payload", &base64_payload)?; - if self.user_headers.is_some() { + let entries: Vec = if self.user_headers.is_some() { let headers_map = self.user_headers_map().map_err(serde::ser::Error::custom)?; - - state.serialize_field("user_headers", &headers_map)?; - } + if let Some(map) = headers_map { + map.into_iter() + .map(|(key, value)| HeaderEntry { key, value }) + .collect() + } else { + Vec::new() + } + } else { + Vec::new() + }; + state.serialize_field("user_headers", &entries)?; state.end() } @@ -567,6 +574,7 @@ impl<'de> Deserialize<'de> for IggyMessage { where D: Deserializer<'de>, { + use super::user_headers::HeaderEntry; use serde::de::{self, MapAccess, Visitor}; use std::fmt; @@ -601,7 +609,12 @@ impl<'de> Deserialize<'de> for IggyMessage { payload = Some(Bytes::from(decoded)); } "user_headers" => { - user_headers = Some(map.next_value()?); + let entries: Vec = map.next_value()?; + let mut headers_map = HashMap::new(); + for entry in entries { + headers_map.insert(entry.key, entry.value); + } + user_headers = Some(headers_map); } _ => { let _ = map.next_value::()?; @@ -668,8 +681,8 @@ mod tests { fn test_create_with_headers() { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("content-type").unwrap(), - HeaderValue::from_str("text/plain").unwrap(), + HeaderKey::try_from("content-type").unwrap(), + HeaderValue::try_from("text/plain").unwrap(), ); let message = IggyMessage::builder() @@ -682,7 +695,7 @@ mod tests { let headers_map = message.user_headers_map().unwrap().unwrap(); assert_eq!(headers_map.len(), 1); - assert!(headers_map.contains_key(&HeaderKey::new("content-type").unwrap())); + assert!(headers_map.contains_key(&HeaderKey::try_from("content-type").unwrap())); } #[test] @@ -720,16 +733,38 @@ mod tests { assert_eq!(original.payload, decoded.payload); } + #[test] + fn test_json_serialization_without_headers() { + let original = IggyMessage::builder() + .id(1) + .payload(Bytes::from("test")) + .build() + .expect("Message creation should not fail"); + + let json = serde_json::to_string(&original).expect("JSON serialization should not fail"); + + assert!(json.contains("\"user_headers\":[]")); + + let deserialized: IggyMessage = + serde_json::from_str(&json).expect("JSON deserialization should not fail"); + + assert_eq!(original.header.id, deserialized.header.id); + assert_eq!(original.payload, deserialized.payload); + + let headers_map = deserialized.user_headers_map().unwrap(); + assert!(headers_map.map(|m| m.is_empty()).unwrap_or(true)); + } + #[test] fn test_json_serialization_with_headers() { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("content-type").unwrap(), - HeaderValue::from_str("text/plain").unwrap(), + HeaderKey::try_from("content-type").unwrap(), + HeaderValue::try_from("text/plain").unwrap(), ); headers.insert( - HeaderKey::new("correlation-id").unwrap(), - HeaderValue::from_str("123456").unwrap(), + HeaderKey::try_from("correlation-id").unwrap(), + HeaderValue::try_from("123456").unwrap(), ); let original = IggyMessage::builder() @@ -773,12 +808,12 @@ mod tests { assert_eq!(original_map.len(), deserialized_map.len()); assert_eq!( - original_map.get(&HeaderKey::new("content-type").unwrap()), - deserialized_map.get(&HeaderKey::new("content-type").unwrap()) + original_map.get(&HeaderKey::try_from("content-type").unwrap()), + deserialized_map.get(&HeaderKey::try_from("content-type").unwrap()) ); assert_eq!( - original_map.get(&HeaderKey::new("correlation-id").unwrap()), - deserialized_map.get(&HeaderKey::new("correlation-id").unwrap()) + original_map.get(&HeaderKey::try_from("correlation-id").unwrap()), + deserialized_map.get(&HeaderKey::try_from("correlation-id").unwrap()) ); } } diff --git a/core/common/src/types/message/mod.rs b/core/common/src/types/message/mod.rs index 5ab0dbf33a..caa29640bd 100644 --- a/core/common/src/types/message/mod.rs +++ b/core/common/src/types/message/mod.rs @@ -55,4 +55,7 @@ pub use partitioning_kind::PartitioningKind; pub use polled_messages::PolledMessages; pub use polling_kind::PollingKind; pub use polling_strategy::PollingStrategy; -pub use user_headers::{HeaderKey, HeaderKind, HeaderValue}; +pub use user_headers::{ + HeaderEntry, HeaderField, HeaderKey, HeaderKind, HeaderValue, KeyMarker, UserHeaders, + ValueMarker, deserialize_headers, serialize_headers, +}; diff --git a/core/common/src/types/message/user_headers.rs b/core/common/src/types/message/user_headers.rs index e5cfa37bf2..aaec4ffe02 100644 --- a/core/common/src/types/message/user_headers.rs +++ b/core/common/src/types/message/user_headers.rs @@ -25,89 +25,211 @@ use serde_with::serde_as; use std::collections::HashMap; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; use std::str::FromStr; -/// Represents a header key with a unique name. The name is case-insensitive and wraps a string. +/// Type alias for header keys in user-defined message headers. +/// +/// Header keys can be created from various types using `From`/`TryFrom` traits: +/// - Fixed-size types (infallible): `bool`, `i8`-`i128`, `u8`-`u128`, `f32`, `f64` +/// - Variable-size types (fallible, max 255 bytes): `&str`, `String`, `&[u8]`, `Vec` +/// +/// Values can be extracted back using `TryFrom` or `as_*` methods. +/// +/// # Examples +/// +/// ``` +/// use iggy_common::{HeaderKey, HeaderValue, IggyError}; +/// +/// // Create from string (most common) +/// let key = HeaderKey::try_from("content-type")?; +/// +/// // Create from integer (for numeric keys) +/// let key: HeaderKey = 42u32.into(); +/// +/// // Extract value back +/// let num: u32 = key.try_into()?; +/// assert_eq!(num, 42); +/// # Ok::<(), IggyError>(()) +/// ``` +pub type HeaderKey = HeaderField; + +/// Type alias for header values in user-defined message headers. +/// +/// Header values can be created from various types using `From`/`TryFrom` traits: +/// - Fixed-size types (infallible): `bool`, `i8`-`i128`, `u8`-`u128`, `f32`, `f64` +/// - Variable-size types (fallible, max 255 bytes): `&str`, `String`, `&[u8]`, `Vec` +/// +/// Values can be extracted back using `TryFrom` or `as_*` methods. +/// +/// # Examples +/// +/// ``` +/// use iggy_common::{HeaderKey, HeaderValue, IggyError}; +/// +/// // Create from various types +/// let str_val = HeaderValue::try_from("text/plain")?; +/// let int_val: HeaderValue = 42i32.into(); +/// let bool_val: HeaderValue = true.into(); +/// let float_val: HeaderValue = 3.14f64.into(); +/// +/// // Extract values back using TryFrom +/// let num: i32 = int_val.try_into()?; +/// assert_eq!(num, 42); +/// +/// // Or use as_* methods +/// let str_val = HeaderValue::try_from("hello")?; +/// assert_eq!(str_val.as_str()?, "hello"); +/// # Ok::<(), IggyError>(()) +/// ``` +pub type HeaderValue = HeaderField; + +/// Type alias for a collection of user-defined message headers. +pub type UserHeaders = HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct KeyMarker; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ValueMarker; + +/// A typed header field that can be used as either a key or value in message headers. +/// +/// `HeaderField` is a generic struct parameterized by a marker type (`KeyMarker` or `ValueMarker`) +/// to distinguish between keys and values at the type level. Use the type aliases +/// [`HeaderKey`] and [`HeaderValue`] instead of using this struct directly. +/// +/// # Creating Header Fields +/// +/// Use `From` trait for fixed-size types (infallible): +/// ``` +/// use iggy_common::HeaderValue; +/// +/// let val: HeaderValue = 42i32.into(); +/// let val: HeaderValue = true.into(); +/// let val: HeaderValue = 3.14f64.into(); +/// ``` +/// +/// Use `TryFrom` trait for variable-size types (fallible, max 255 bytes): +/// ``` +/// use iggy_common::{HeaderValue, IggyError}; +/// +/// let val = HeaderValue::try_from("hello")?; +/// let val = HeaderValue::try_from(vec![1u8, 2, 3])?; +/// # Ok::<(), IggyError>(()) +/// ``` +/// +/// # Extracting Values +/// +/// Use `TryFrom` to extract values (returns error if kind doesn't match): +/// ``` +/// use iggy_common::{HeaderValue, IggyError}; +/// +/// let val: HeaderValue = 42i32.into(); +/// let num: i32 = val.try_into()?; +/// +/// // Using reference to avoid consuming the value +/// let val: HeaderValue = 100u64.into(); +/// let num: u64 = (&val).try_into()?; +/// // val is still usable here +/// # Ok::<(), IggyError>(()) +/// ``` +/// +/// Or use `as_*` methods: +/// ``` +/// use iggy_common::{HeaderValue, IggyError}; +/// +/// let val: HeaderValue = 42i32.into(); +/// assert_eq!(val.as_int32()?, 42); +/// # Ok::<(), IggyError>(()) +/// ``` +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct HeaderKey(String); - -impl HeaderKey { - pub fn new(key: &str) -> Result { - if key.is_empty() || key.len() > 255 { - return Err(IggyError::InvalidHeaderKey); - } - - Ok(Self(key.to_lowercase().to_string())) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl Display for HeaderKey { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } +pub struct HeaderField { + kind: HeaderKind, + #[serde_as(as = "Base64")] + value: Bytes, + #[serde(skip)] + _marker: PhantomData, } -impl Hash for HeaderKey { - fn hash(&self, state: &mut H) { - self.0.hash(state); +impl HeaderField { + /// Returns the kind of this header field. + pub fn kind(&self) -> HeaderKind { + self.kind } -} -impl FromStr for HeaderKey { - type Err = IggyError; - fn from_str(s: &str) -> Result { - Self::new(s) + /// Returns a clone of the raw bytes value. + pub fn value(&self) -> Bytes { + self.value.clone() } -} -impl TryFrom<&str> for HeaderKey { - type Error = IggyError; - fn try_from(value: &str) -> Result { - Self::new(value) + /// Returns a reference to the raw bytes value. + pub fn as_bytes(&self) -> &[u8] { + &self.value } } -/// Represents a header value of a specific kind. -/// It consists of the following fields: -/// - `kind`: the kind of the header value. -/// - `value`: the value of the header. -#[serde_as] -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct HeaderValue { - /// The kind of the header value. - pub kind: HeaderKind, - /// The binary value of the header payload. - #[serde_as(as = "Base64")] - pub value: Bytes, -} - -/// Represents the kind of a header value. +/// Indicates the type of value stored in a [`HeaderField`]. +/// +/// This enum is used to track what type of data is stored in the header's raw bytes, +/// enabling proper deserialization and type checking when extracting values. +/// +/// # Supported Types +/// +/// | Kind | Rust Type | Size (bytes) | +/// |------|-----------|--------------| +/// | `Raw` | `&[u8]` / `Vec` | 1-255 | +/// | `String` | `&str` / `String` | 1-255 | +/// | `Bool` | `bool` | 1 | +/// | `Int8` | `i8` | 1 | +/// | `Int16` | `i16` | 2 | +/// | `Int32` | `i32` | 4 | +/// | `Int64` | `i64` | 8 | +/// | `Int128` | `i128` | 16 | +/// | `Uint8` | `u8` | 1 | +/// | `Uint16` | `u16` | 2 | +/// | `Uint32` | `u32` | 4 | +/// | `Uint64` | `u64` | 8 | +/// | `Uint128` | `u128` | 16 | +/// | `Float32` | `f32` | 4 | +/// | `Float64` | `f64` | 8 | #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum HeaderKind { + /// Raw binary data. Raw, + /// UTF-8 encoded string. String, + /// Boolean value. Bool, + /// Signed 8-bit integer. Int8, + /// Signed 16-bit integer. Int16, + /// Signed 32-bit integer. Int32, + /// Signed 64-bit integer. Int64, + /// Signed 128-bit integer. Int128, + /// Unsigned 8-bit integer. Uint8, + /// Unsigned 16-bit integer. Uint16, + /// Unsigned 32-bit integer. Uint32, + /// Unsigned 64-bit integer. Uint64, + /// Unsigned 128-bit integer. Uint128, + /// 32-bit floating point number. Float32, + /// 64-bit floating point number. Float64, } impl HeaderKind { - /// Returns the code of the header kind. pub fn as_code(&self) -> u8 { match self { HeaderKind::Raw => 1, @@ -128,7 +250,6 @@ impl HeaderKind { } } - /// Returns the header kind from the code. pub fn from_code(code: u8) -> Result { match code { 1 => Ok(HeaderKind::Raw), @@ -149,6 +270,17 @@ impl HeaderKind { _ => Err(IggyError::InvalidCommand), } } + + fn expected_size(&self) -> Option { + match self { + HeaderKind::Raw | HeaderKind::String => None, + HeaderKind::Bool | HeaderKind::Int8 | HeaderKind::Uint8 => Some(1), + HeaderKind::Int16 | HeaderKind::Uint16 => Some(2), + HeaderKind::Int32 | HeaderKind::Uint32 | HeaderKind::Float32 => Some(4), + HeaderKind::Int64 | HeaderKind::Uint64 | HeaderKind::Float64 => Some(8), + HeaderKind::Int128 | HeaderKind::Uint128 => Some(16), + } + } } impl FromStr for HeaderKind { @@ -175,13 +307,6 @@ impl FromStr for HeaderKind { } } -impl Display for HeaderValue { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: ", self.kind)?; - write!(f, "{}", self.value_only_to_string()) - } -} - impl Display for HeaderKind { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { match *self { @@ -204,1055 +329,1077 @@ impl Display for HeaderKind { } } -impl FromStr for HeaderValue { - type Err = IggyError; - fn from_str(s: &str) -> Result { - Self::from(HeaderKind::String, s.as_bytes()) +impl Display for HeaderField { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: ", self.kind)?; + write!(f, "{}", self.to_string_value()) } } -impl HeaderValue { - /// Creates a new header value from the specified kind and value. - /// The kind is parsed from the string representation. - /// The value is parsed from the string representation. - pub fn from_kind_str_and_value_str(kind: &str, value: &str) -> Result { - let kind = HeaderKind::from_str(kind)?; - Self::from_kind_and_value_str(kind, value) - } - - /// Creates a new header value from the specified kind and value. - /// The value is parsed from the string representation. - pub fn from_kind_and_value_str(kind: HeaderKind, value: &str) -> Result { - match kind { - HeaderKind::Raw => Self::from_raw(value.as_bytes()), - HeaderKind::String => Self::from_str(value), - HeaderKind::Bool => { - Self::from_bool(value.parse().map_err(|_| IggyError::InvalidBooleanValue)?) - } - HeaderKind::Int8 => { - Self::from_int8(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Int16 => { - Self::from_int16(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Int32 => { - Self::from_int32(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Int64 => { - Self::from_int64(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Int128 => { - Self::from_int128(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Uint8 => { - Self::from_uint8(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Uint16 => { - Self::from_uint16(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Uint32 => { - Self::from_uint32(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Uint64 => { - Self::from_uint64(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Uint128 => { - Self::from_uint128(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Float32 => { - Self::from_float32(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - HeaderKind::Float64 => { - Self::from_float64(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) - } - } +impl Hash for HeaderField { + fn hash(&self, state: &mut H) { + self.kind.hash(state); + self.value.hash(state); } - /// Creates a new header value from the specified raw bytes. - pub fn from_raw(value: &[u8]) -> Result { - Self::from(HeaderKind::Raw, value) +} + +impl FromStr for HeaderField { + type Err = IggyError; + fn from_str(s: &str) -> Result { + Self::try_from(s) } +} - /// Returns the raw bytes of the header value. +impl HeaderField { pub fn as_raw(&self) -> Result<&[u8], IggyError> { if self.kind != HeaderKind::Raw { return Err(IggyError::InvalidHeaderValue); } - Ok(&self.value) } - /// Returns the string representation of the header value. pub fn as_str(&self) -> Result<&str, IggyError> { if self.kind != HeaderKind::String { return Err(IggyError::InvalidHeaderValue); } - std::str::from_utf8(&self.value).map_err(|_| IggyError::InvalidUtf8) } - /// Creates a new header value from the specified string. - pub fn from_bool(value: bool) -> Result { - Self::from(HeaderKind::Bool, if value { &[1] } else { &[0] }) - } - - /// Returns the boolean representation of the header value. pub fn as_bool(&self) -> Result { if self.kind != HeaderKind::Bool { return Err(IggyError::InvalidHeaderValue); } - - match self.value[0] { + let bytes: [u8; 1] = self + .value + .as_ref() + .try_into() + .map_err(|_| IggyError::InvalidHeaderValue)?; + match bytes[0] { 0 => Ok(false), 1 => Ok(true), _ => Err(IggyError::InvalidHeaderValue), } } - /// Creates a new header value from the specified boolean. - pub fn from_int8(value: i8) -> Result { - Self::from(HeaderKind::Int8, &value.to_le_bytes()) - } - - /// Returns the i8 representation of the header value. pub fn as_int8(&self) -> Result { if self.kind != HeaderKind::Int8 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(i8::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified i8. - pub fn from_int16(value: i16) -> Result { - Self::from(HeaderKind::Int16, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(i8::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the i16 representation of the header value. pub fn as_int16(&self) -> Result { if self.kind != HeaderKind::Int16 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(i16::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified i16. - pub fn from_int32(value: i32) -> Result { - Self::from(HeaderKind::Int32, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(i16::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the i32 representation of the header value. pub fn as_int32(&self) -> Result { if self.kind != HeaderKind::Int32 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(i32::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified i32. - pub fn from_int64(value: i64) -> Result { - Self::from(HeaderKind::Int64, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(i32::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the i64 representation of the header value. pub fn as_int64(&self) -> Result { if self.kind != HeaderKind::Int64 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(i64::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified i128. - pub fn from_int128(value: i128) -> Result { - Self::from(HeaderKind::Int128, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(i64::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the i128 representation of the header value. pub fn as_int128(&self) -> Result { if self.kind != HeaderKind::Int128 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(i128::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified u8. - pub fn from_uint8(value: u8) -> Result { - Self::from(HeaderKind::Uint8, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(i128::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the u8 representation of the header value. pub fn as_uint8(&self) -> Result { if self.kind != HeaderKind::Uint8 { return Err(IggyError::InvalidHeaderValue); } - - Ok(self.value[0]) + self.value + .as_ref() + .try_into() + .map(u8::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Creates a new header value from the specified u16. - pub fn from_uint16(value: u16) -> Result { - Self::from(HeaderKind::Uint16, &value.to_le_bytes()) - } - - /// Returns the u16 representation of the header value. pub fn as_uint16(&self) -> Result { if self.kind != HeaderKind::Uint16 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(u16::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified u32. - pub fn from_uint32(value: u32) -> Result { - Self::from(HeaderKind::Uint32, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(u16::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the u32 representation of the header value. pub fn as_uint32(&self) -> Result { if self.kind != HeaderKind::Uint32 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(u32::from_le_bytes(value.unwrap())) + self.value + .as_ref() + .try_into() + .map(u32::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Creates a new header value from the specified u64. - pub fn from_uint64(value: u64) -> Result { - Self::from(HeaderKind::Uint64, &value.to_le_bytes()) - } - - /// Returns the u64 representation of the header value. pub fn as_uint64(&self) -> Result { if self.kind != HeaderKind::Uint64 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(u64::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified u128. - pub fn from_uint128(value: u128) -> Result { - Self::from(HeaderKind::Uint128, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(u64::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the u128 representation of the header value. pub fn as_uint128(&self) -> Result { if self.kind != HeaderKind::Uint128 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(u128::from_le_bytes(value.unwrap())) + self.value + .as_ref() + .try_into() + .map(u128::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Creates a new header value from the specified f32. - pub fn from_float32(value: f32) -> Result { - Self::from(HeaderKind::Float32, &value.to_le_bytes()) - } - - /// Returns the f32 representation of the header value. pub fn as_float32(&self) -> Result { if self.kind != HeaderKind::Float32 { return Err(IggyError::InvalidHeaderValue); } - - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); - } - - Ok(f32::from_le_bytes(value.unwrap())) - } - - /// Creates a new header value from the specified f64. - pub fn from_float64(value: f64) -> Result { - Self::from(HeaderKind::Float64, &value.to_le_bytes()) + self.value + .as_ref() + .try_into() + .map(f32::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) } - /// Returns the f64 representation of the header value. pub fn as_float64(&self) -> Result { if self.kind != HeaderKind::Float64 { return Err(IggyError::InvalidHeaderValue); } + self.value + .as_ref() + .try_into() + .map(f64::from_le_bytes) + .map_err(|_| IggyError::InvalidHeaderValue) + } - let value = self.value.to_vec().try_into(); - if value.is_err() { - return Err(IggyError::InvalidHeaderValue); + pub fn to_string_value(&self) -> String { + match self.kind { + HeaderKind::Raw => format!("{:?}", self.value), + HeaderKind::String => String::from_utf8_lossy(&self.value).to_string(), + HeaderKind::Bool => { + if self.value.is_empty() { + "".to_string() + } else { + format!("{}", self.value[0] != 0) + } + } + HeaderKind::Int8 => self + .value + .as_ref() + .try_into() + .map(|b| i8::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Int16 => self + .value + .as_ref() + .try_into() + .map(|b| i16::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Int32 => self + .value + .as_ref() + .try_into() + .map(|b| i32::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Int64 => self + .value + .as_ref() + .try_into() + .map(|b| i64::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Int128 => self + .value + .as_ref() + .try_into() + .map(|b| i128::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Uint8 => self + .value + .as_ref() + .try_into() + .map(|b| u8::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Uint16 => self + .value + .as_ref() + .try_into() + .map(|b| u16::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Uint32 => self + .value + .as_ref() + .try_into() + .map(|b| u32::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Uint64 => self + .value + .as_ref() + .try_into() + .map(|b| u64::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Uint128 => self + .value + .as_ref() + .try_into() + .map(|b| u128::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Float32 => self + .value + .as_ref() + .try_into() + .map(|b| f32::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), + HeaderKind::Float64 => self + .value + .as_ref() + .try_into() + .map(|b| f64::from_le_bytes(b).to_string()) + .unwrap_or_else(|_| "".to_string()), } + } - Ok(f64::from_le_bytes(value.unwrap())) + fn new_unchecked(kind: HeaderKind, value: &[u8]) -> Self { + Self { + kind, + value: Bytes::from(value.to_vec()), + _marker: PhantomData, + } } +} - /// Creates a new header value from the specified kind and value. - fn from(kind: HeaderKind, value: &[u8]) -> Result { +impl TryFrom<&str> for HeaderField { + type Error = IggyError; + fn try_from(value: &str) -> Result { if value.is_empty() || value.len() > 255 { return Err(IggyError::InvalidHeaderValue); } + Ok(Self::new_unchecked(HeaderKind::String, value.as_bytes())) + } +} - Ok(Self { - kind, - value: Bytes::from(value.to_vec()), - }) +impl TryFrom for HeaderField { + type Error = IggyError; + fn try_from(value: String) -> Result { + Self::try_from(value.as_str()) } +} - /// Returns the string representation of the header value without the kind. - pub fn value_only_to_string(&self) -> String { - match self.kind { - HeaderKind::Raw => format!("{:?}", self.value), - HeaderKind::String => format!("{}", String::from_utf8_lossy(&self.value)), - HeaderKind::Bool => format!("{}", self.value[0] != 0), - HeaderKind::Int8 => format!( - "{}", - i8::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Int16 => format!( - "{}", - i16::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Int32 => format!( - "{}", - i32::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Int64 => format!( - "{}", - i64::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Int128 => format!( - "{}", - i128::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Uint8 => format!( - "{}", - u8::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Uint16 => format!( - "{}", - u16::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Uint32 => format!( - "{}", - u32::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Uint64 => format!( - "{}", - u64::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Uint128 => format!( - "{}", - u128::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Float32 => format!( - "{}", - f32::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), - HeaderKind::Float64 => format!( - "{}", - f64::from_le_bytes(self.value.to_vec().try_into().unwrap()) - ), +impl TryFrom<&[u8]> for HeaderField { + type Error = IggyError; + fn try_from(value: &[u8]) -> Result { + if value.is_empty() || value.len() > 255 { + return Err(IggyError::InvalidHeaderValue); } + Ok(Self::new_unchecked(HeaderKind::Raw, value)) } } -impl BytesSerializable for HashMap { - fn to_bytes(&self) -> Bytes { - if self.is_empty() { - return Bytes::new(); - } +impl TryFrom> for HeaderField { + type Error = IggyError; + fn try_from(value: Vec) -> Result { + Self::try_from(value.as_slice()) + } +} - let mut bytes = BytesMut::new(); - for (key, value) in self { - #[allow(clippy::cast_possible_truncation)] - bytes.put_u32_le(key.0.len() as u32); - bytes.put_slice(key.0.as_bytes()); - bytes.put_u8(value.kind.as_code()); - #[allow(clippy::cast_possible_truncation)] - bytes.put_u32_le(value.value.len() as u32); - bytes.put_slice(&value.value); - } +impl From for HeaderField { + fn from(value: bool) -> Self { + Self::new_unchecked(HeaderKind::Bool, if value { &[1] } else { &[0] }) + } +} - bytes.freeze() +impl From for HeaderField { + fn from(value: i8) -> Self { + Self::new_unchecked(HeaderKind::Int8, &value.to_le_bytes()) } +} - fn from_bytes(bytes: Bytes) -> Result - where - Self: Sized, - { - if bytes.is_empty() { - return Ok(Self::new()); - } +impl From for HeaderField { + fn from(value: i16) -> Self { + Self::new_unchecked(HeaderKind::Int16, &value.to_le_bytes()) + } +} - let mut headers = Self::new(); - let mut position = 0; - while position < bytes.len() { - let key_length = u32::from_le_bytes( - bytes[position..position + 4] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize; - if key_length == 0 || key_length > 255 { - tracing::error!("Invalid header key length: {key_length}"); - return Err(IggyError::InvalidHeaderKey); - } - position += 4; - let key = match String::from_utf8(bytes[position..position + key_length].to_vec()) { - Ok(k) => k, - Err(e) => { - tracing::error!("Invalid header key: {e}"); - return Err(IggyError::InvalidHeaderKey); - } - }; - position += key_length; - let kind = HeaderKind::from_code(bytes[position])?; - position += 1; - let value_length = u32::from_le_bytes( - bytes[position..position + 4] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize; - if value_length == 0 || value_length > 255 { - tracing::error!("Invalid header value length: {value_length}"); - return Err(IggyError::InvalidHeaderValue); - } - position += 4; - let value = bytes[position..position + value_length].to_vec(); - position += value_length; - headers.insert( - HeaderKey(key), - HeaderValue { - kind, - value: Bytes::from(value), - }, - ); - } +impl From for HeaderField { + fn from(value: i32) -> Self { + Self::new_unchecked(HeaderKind::Int32, &value.to_le_bytes()) + } +} - Ok(headers) +impl From for HeaderField { + fn from(value: i64) -> Self { + Self::new_unchecked(HeaderKind::Int64, &value.to_le_bytes()) } } -/// Returns the size in bytes of the specified headers. -pub fn get_user_headers_size(headers: &Option>) -> Option { - let mut size = 0; - if let Some(headers) = headers { - for (key, value) in headers { - size += 4 + key.as_str().len() as u32 + 1 + 4 + value.value.len() as u32; - } +impl From for HeaderField { + fn from(value: i128) -> Self { + Self::new_unchecked(HeaderKind::Int128, &value.to_le_bytes()) } - Some(size) } -#[cfg(test)] -mod tests { - use super::*; +impl From for HeaderField { + fn from(value: u8) -> Self { + Self::new_unchecked(HeaderKind::Uint8, &value.to_le_bytes()) + } +} - #[test] - fn header_key_should_be_created_for_valid_value() { - let value = "key-1"; - let header_key = HeaderKey::new(value); - assert!(header_key.is_ok()); - assert_eq!(header_key.unwrap().0, value); +impl From for HeaderField { + fn from(value: u16) -> Self { + Self::new_unchecked(HeaderKind::Uint16, &value.to_le_bytes()) } +} - #[test] - fn header_key_should_not_be_created_for_empty_value() { - let value = ""; - let header_key = HeaderKey::new(value); - assert!(header_key.is_err()); - let error = header_key.unwrap_err(); - assert_eq!(error.as_code(), IggyError::InvalidHeaderKey.as_code()); +impl From for HeaderField { + fn from(value: u32) -> Self { + Self::new_unchecked(HeaderKind::Uint32, &value.to_le_bytes()) } +} - #[test] - fn header_key_should_not_be_created_for_too_long_value() { - let value = "a".repeat(256); - let header_key = HeaderKey::new(&value); - assert!(header_key.is_err()); - let error = header_key.unwrap_err(); - assert_eq!(error.as_code(), IggyError::InvalidHeaderKey.as_code()); +impl From for HeaderField { + fn from(value: u64) -> Self { + Self::new_unchecked(HeaderKind::Uint64, &value.to_le_bytes()) } +} - #[test] - fn header_value_should_not_be_created_for_empty_value() { - let header_value = HeaderValue::from(HeaderKind::Raw, &[]); - assert!(header_value.is_err()); - let error = header_value.unwrap_err(); - assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); +impl From for HeaderField { + fn from(value: u128) -> Self { + Self::new_unchecked(HeaderKind::Uint128, &value.to_le_bytes()) } +} - #[test] - fn header_value_should_not_be_created_for_too_long_value() { - let value = b"a".repeat(256); - let header_value = HeaderValue::from(HeaderKind::Raw, &value); - assert!(header_value.is_err()); - let error = header_value.unwrap_err(); - assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); +impl From for HeaderField { + fn from(value: f32) -> Self { + Self::new_unchecked(HeaderKind::Float32, &value.to_le_bytes()) } +} - #[test] - fn header_value_should_be_created_from_raw_bytes() { - let value = b"Value 1"; - let header_value = HeaderValue::from_raw(value); - assert!(header_value.is_ok()); - assert_eq!(header_value.unwrap().value.as_ref(), value); +impl From for HeaderField { + fn from(value: f64) -> Self { + Self::new_unchecked(HeaderKind::Float64, &value.to_le_bytes()) } +} - #[test] - fn header_value_should_be_created_from_str() { - let value = "Value 1"; - let header_value = HeaderValue::from_str(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::String); - assert_eq!(header_value.value, value.as_bytes()); - assert_eq!(header_value.as_str().unwrap(), value); +impl TryFrom> for bool { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_bool() } +} - #[test] - fn header_value_should_be_created_from_bool() { - let value = true; - let header_value = HeaderValue::from_bool(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Bool); - assert_eq!(header_value.value.as_ref(), if value { [1] } else { [0] }); - assert_eq!(header_value.as_bool().unwrap(), value); +impl TryFrom<&HeaderField> for bool { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_bool() } +} - #[test] - fn header_value_should_be_created_from_int8() { - let value = 123; - let header_value = HeaderValue::from_int8(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Int8); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_int8().unwrap(), value); +impl TryFrom> for i8 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_int8() } +} - #[test] - fn header_value_should_be_created_from_int16() { - let value = 12345; - let header_value = HeaderValue::from_int16(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Int16); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_int16().unwrap(), value); +impl TryFrom<&HeaderField> for i8 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_int8() } +} - #[test] - fn header_value_should_be_created_from_int32() { - let value = 123_456; - let header_value = HeaderValue::from_int32(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Int32); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_int32().unwrap(), value); +impl TryFrom> for i16 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_int16() } +} - #[test] - fn header_value_should_be_created_from_int64() { - let value = 123_4567; - let header_value = HeaderValue::from_int64(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Int64); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_int64().unwrap(), value); +impl TryFrom<&HeaderField> for i16 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_int16() } +} - #[test] - fn header_value_should_be_created_from_int128() { - let value = 1234_5678; - let header_value = HeaderValue::from_int128(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Int128); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_int128().unwrap(), value); +impl TryFrom> for i32 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_int32() } +} - #[test] - fn header_value_should_be_created_from_uint8() { - let value = 123; - let header_value = HeaderValue::from_uint8(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Uint8); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_uint8().unwrap(), value); +impl TryFrom<&HeaderField> for i32 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_int32() } +} - #[test] - fn header_value_should_be_created_from_uint16() { - let value = 12345; - let header_value = HeaderValue::from_uint16(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Uint16); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_uint16().unwrap(), value); +impl TryFrom> for i64 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_int64() } +} - #[test] - fn header_value_should_be_created_from_uint32() { - let value = 123_456; - let header_value = HeaderValue::from_uint32(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Uint32); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_uint32().unwrap(), value); +impl TryFrom<&HeaderField> for i64 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_int64() + } +} + +impl TryFrom> for i128 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_int128() + } +} + +impl TryFrom<&HeaderField> for i128 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_int128() + } +} + +impl TryFrom> for u8 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_uint8() + } +} + +impl TryFrom<&HeaderField> for u8 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_uint8() + } +} + +impl TryFrom> for u16 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_uint16() + } +} + +impl TryFrom<&HeaderField> for u16 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_uint16() + } +} + +impl TryFrom> for u32 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_uint32() + } +} + +impl TryFrom<&HeaderField> for u32 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_uint32() + } +} + +impl TryFrom> for u64 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_uint64() } +} + +impl TryFrom<&HeaderField> for u64 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_uint64() + } +} + +impl TryFrom> for u128 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_uint128() + } +} + +impl TryFrom<&HeaderField> for u128 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_uint128() + } +} + +impl TryFrom> for f32 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_float32() + } +} + +impl TryFrom<&HeaderField> for f32 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_float32() + } +} + +impl TryFrom> for f64 { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_float64() + } +} + +impl TryFrom<&HeaderField> for f64 { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_float64() + } +} + +impl TryFrom> for String { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_str().map(|s| s.to_owned()) + } +} + +impl TryFrom<&HeaderField> for String { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_str().map(|s| s.to_owned()) + } +} + +impl TryFrom> for Vec { + type Error = IggyError; + fn try_from(field: HeaderField) -> Result { + field.as_raw().map(|s| s.to_vec()) + } +} + +impl TryFrom<&HeaderField> for Vec { + type Error = IggyError; + fn try_from(field: &HeaderField) -> Result { + field.as_raw().map(|s| s.to_vec()) + } +} + +impl BytesSerializable for HashMap { + fn to_bytes(&self) -> Bytes { + if self.is_empty() { + return Bytes::new(); + } + + let mut bytes = BytesMut::new(); + for (key, value) in self { + bytes.put_u8(key.kind().as_code()); + #[allow(clippy::cast_possible_truncation)] + bytes.put_u32_le(key.as_bytes().len() as u32); + bytes.put_slice(key.as_bytes()); + bytes.put_u8(value.kind().as_code()); + #[allow(clippy::cast_possible_truncation)] + bytes.put_u32_le(value.as_bytes().len() as u32); + bytes.put_slice(value.as_bytes()); + } + + bytes.freeze() + } + + fn from_bytes(bytes: Bytes) -> Result + where + Self: Sized, + { + if bytes.is_empty() { + return Ok(Self::new()); + } + + let mut headers = Self::new(); + let mut position = 0; + while position < bytes.len() { + let key_kind = HeaderKind::from_code(bytes[position])?; + position += 1; + + if position + 4 > bytes.len() { + return Err(IggyError::InvalidHeaderKey); + } + let key_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; + if key_length == 0 || key_length > 255 { + return Err(IggyError::InvalidHeaderKey); + } + position += 4; + + if position + key_length > bytes.len() { + return Err(IggyError::InvalidHeaderKey); + } + if let Some(expected) = key_kind.expected_size() + && key_length != expected + { + return Err(IggyError::InvalidHeaderKey); + } + let key_value = bytes[position..position + key_length].to_vec(); + position += key_length; + + if position >= bytes.len() { + return Err(IggyError::InvalidHeaderValue); + } + let value_kind = HeaderKind::from_code(bytes[position])?; + position += 1; + + if position + 4 > bytes.len() { + return Err(IggyError::InvalidHeaderValue); + } + let value_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; + if value_length == 0 || value_length > 255 { + return Err(IggyError::InvalidHeaderValue); + } + position += 4; + + if position + value_length > bytes.len() { + return Err(IggyError::InvalidHeaderValue); + } + if let Some(expected) = value_kind.expected_size() + && value_length != expected + { + return Err(IggyError::InvalidHeaderValue); + } + let value_value = bytes[position..position + value_length].to_vec(); + position += value_length; + + headers.insert( + HeaderKey::new_unchecked(key_kind, &key_value), + HeaderValue::new_unchecked(value_kind, &value_value), + ); + } + + Ok(headers) + } +} + +pub fn get_user_headers_size(headers: &Option>) -> Option { + let mut size = 0; + if let Some(headers) = headers { + for (key, value) in headers { + size += 1 + 4 + key.as_bytes().len() as u32 + 1 + 4 + value.as_bytes().len() as u32; + } + } + Some(size) +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct HeaderEntry { + pub key: HeaderKey, + pub value: HeaderValue, +} + +pub fn serialize_headers(headers: &Option, serializer: S) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeSeq; + + match headers { + Some(map) => { + let mut seq = serializer.serialize_seq(Some(map.len()))?; + for (key, value) in map { + seq.serialize_element(&HeaderEntry { + key: key.clone(), + value: value.clone(), + })?; + } + seq.end() + } + None => serializer.serialize_none(), + } +} + +pub fn deserialize_headers<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let entries: Option> = Option::deserialize(deserializer)?; + match entries { + Some(vec) => { + let mut map = UserHeaders::new(); + for entry in vec { + map.insert(entry.key, entry.value); + } + Ok(Some(map)) + } + None => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; #[test] - fn header_value_should_be_created_from_uint64() { - let value = 123_4567; - let header_value = HeaderValue::from_uint64(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Uint64); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_uint64().unwrap(), value); + fn header_key_should_be_created_for_valid_value() { + let value = "key-1"; + let header_key = HeaderKey::try_from(value); + assert!(header_key.is_ok()); + let header_key = header_key.unwrap(); + assert_eq!(header_key.kind, HeaderKind::String); + assert_eq!(header_key.as_str().unwrap(), value); } #[test] - fn header_value_should_be_created_from_uint128() { - let value = 1234_5678; - let header_value = HeaderValue::from_uint128(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Uint128); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_uint128().unwrap(), value); + fn header_key_should_not_be_created_for_empty_value() { + let value = ""; + let header_key = HeaderKey::try_from(value); + assert!(header_key.is_err()); + let error = header_key.unwrap_err(); + assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); } #[test] - fn header_value_should_be_created_from_float32() { - let value = 123.01; - let header_value = HeaderValue::from_float32(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Float32); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_float32().unwrap(), value); + fn header_key_should_not_be_created_for_too_long_value() { + let value = "a".repeat(256); + let header_key = HeaderKey::try_from(value.as_str()); + assert!(header_key.is_err()); + let error = header_key.unwrap_err(); + assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); } #[test] - fn header_value_should_be_created_from_float64() { - let value = 1234.01234; - let header_value = HeaderValue::from_float64(value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Float64); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_float64().unwrap(), value); + fn header_key_should_be_created_from_int32() { + let value = 12345i32; + let header_key: HeaderKey = value.into(); + assert_eq!(header_key.kind, HeaderKind::Int32); + assert_eq!(header_key.as_int32().unwrap(), value); } #[test] - fn header_value_should_be_created_string_from_kind_and_value_str() { - let value = "Value 1"; - let header_value = HeaderValue::from_kind_str_and_value_str("string", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::String); - assert_eq!(header_value.value, value.as_bytes()); - assert_eq!(header_value.as_str().unwrap(), value); + fn header_value_should_not_be_created_for_empty_value() { + let header_value = HeaderValue::try_from([].as_slice()); + assert!(header_value.is_err()); + let error = header_value.unwrap_err(); + assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); } #[test] - fn header_value_should_be_created_float_from_kind_and_value_str() { - let value: f64 = 1234.01234; - let header_value = HeaderValue::from_kind_str_and_value_str("float64", &value.to_string()); + fn header_value_should_not_be_created_for_too_long_value() { + let value = b"a".repeat(256); + let header_value = HeaderValue::try_from(value.as_slice()); + assert!(header_value.is_err()); + let error = header_value.unwrap_err(); + assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); + } + + #[test] + fn header_value_should_be_created_from_raw_bytes() { + let value = b"Value 1"; + let header_value = HeaderValue::try_from(value.as_slice()); assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Float64); - assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); - assert_eq!(header_value.as_float64().unwrap(), value); + assert_eq!(header_value.unwrap().value.as_ref(), value); } #[test] - fn header_value_should_be_created_raw_from_kind_and_value_str() { + fn header_value_should_be_created_from_str() { let value = "Value 1"; - let header_value = HeaderValue::from_kind_str_and_value_str("raw", value); + let header_value = HeaderValue::from_str(value); assert!(header_value.is_ok()); let header_value = header_value.unwrap(); - assert_eq!(header_value.kind, HeaderKind::Raw); + assert_eq!(header_value.kind, HeaderKind::String); assert_eq!(header_value.value, value.as_bytes()); + assert_eq!(header_value.as_str().unwrap(), value); } #[test] - fn header_value_should_be_created_bool_from_kind_and_value_str() { - let value = "true"; - let header_value = HeaderValue::from_kind_str_and_value_str("bool", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_bool() { + let value = true; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Bool); - assert_eq!(header_value.value, vec![1]); - assert!(header_value.as_bool().unwrap()); + assert_eq!(header_value.value.as_ref(), if value { [1] } else { [0] }); + assert_eq!(header_value.as_bool().unwrap(), value); } #[test] - fn header_value_should_be_created_int8_from_kind_and_value_str() { - let value = "123"; - let header_value = HeaderValue::from_kind_str_and_value_str("int8", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_int8() { + let value: i8 = 123; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Int8); - assert_eq!(header_value.value, vec![123]); - assert_eq!(header_value.as_int8().unwrap(), 123); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_int8().unwrap(), value); } #[test] - fn header_value_should_be_created_int16_from_kind_and_value_str() { - let value = "1234"; - let header_value = HeaderValue::from_kind_str_and_value_str("int16", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_int16() { + let value: i16 = 12345; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Int16); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_int16().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_int16().unwrap(), value); } #[test] - fn header_value_should_be_created_int32_from_kind_and_value_str() { - let value = "123456"; - let header_value = HeaderValue::from_kind_str_and_value_str("int32", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_int32() { + let value: i32 = 123_456; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Int32); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_int32().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_int32().unwrap(), value); } #[test] - fn header_value_should_be_created_int64_from_kind_and_value_str() { - let value = "123456789"; - let header_value = HeaderValue::from_kind_str_and_value_str("int64", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_int64() { + let value: i64 = 123_4567; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Int64); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_int64().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_int64().unwrap(), value); } #[test] - fn header_value_should_be_created_int128_from_kind_and_value_str() { - let value = "123456789123456789"; - let header_value = HeaderValue::from_kind_str_and_value_str("int128", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_int128() { + let value: i128 = 1234_5678; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Int128); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_int128().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_int128().unwrap(), value); } #[test] - fn header_value_should_be_created_uint8_from_kind_and_value_str() { - let value = "123"; - let header_value = HeaderValue::from_kind_str_and_value_str("uint8", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_uint8() { + let value: u8 = 123; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Uint8); - assert_eq!(header_value.value, vec![123]); - assert_eq!(header_value.as_uint8().unwrap(), 123); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_uint8().unwrap(), value); } #[test] - fn header_value_should_be_created_uint16_from_kind_and_value_str() { - let value = "12345"; - let header_value = HeaderValue::from_kind_str_and_value_str("uint16", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_uint16() { + let value: u16 = 12345; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Uint16); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_uint16().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_uint16().unwrap(), value); } #[test] - fn header_value_should_be_created_uint32_from_kind_and_value_str() { - let value = "123456"; - let header_value = HeaderValue::from_kind_str_and_value_str("uint32", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_uint32() { + let value: u32 = 123_456; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Uint32); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_uint32().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_uint32().unwrap(), value); } #[test] - fn header_value_should_be_created_uint64_from_kind_and_value_str() { - let value = "123456789"; - let header_value = HeaderValue::from_kind_str_and_value_str("uint64", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_uint64() { + let value: u64 = 123_4567; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Uint64); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_uint64().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_uint64().unwrap(), value); } #[test] - fn header_value_should_be_created_uint128_from_kind_and_value_str() { - let value = "123456789123456789"; - let header_value = HeaderValue::from_kind_str_and_value_str("uint128", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_uint128() { + let value: u128 = 1234_5678; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Uint128); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_uint128().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_uint128().unwrap(), value); } #[test] - fn header_value_should_be_created_float32_from_kind_and_value_str() { - let value = "123.01"; - let header_value = HeaderValue::from_kind_str_and_value_str("float32", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_float32() { + let value: f32 = 123.01; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Float32); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_float32().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_float32().unwrap(), value); } #[test] - fn header_value_should_be_created_float64_from_kind_and_value_str() { - let value = "1234.01234"; - let header_value = HeaderValue::from_kind_str_and_value_str("float64", value); - assert!(header_value.is_ok()); - let header_value = header_value.unwrap(); + fn header_value_should_be_created_from_float64() { + let value: f64 = 1234.01234; + let header_value: HeaderValue = value.into(); assert_eq!(header_value.kind, HeaderKind::Float64); - assert_eq!( - header_value.value.as_ref(), - value.parse::().unwrap().to_le_bytes() - ); - assert_eq!( - header_value.as_float64().unwrap(), - value.parse::().unwrap() - ); + assert_eq!(header_value.value.as_ref(), value.to_le_bytes()); + assert_eq!(header_value.as_float64().unwrap(), value); } #[test] - fn value_only_to_string_for_string_kind() { + fn to_string_value_for_string_kind() { let header_value = HeaderValue::from_str("Hello").unwrap(); - assert_eq!(header_value.value_only_to_string(), "Hello"); + assert_eq!(header_value.to_string_value(), "Hello"); } #[test] - fn value_only_to_string_for_bool_kind() { - let header_value = HeaderValue::from_bool(true).unwrap(); - assert_eq!(header_value.value_only_to_string(), "true"); + fn to_string_value_for_bool_kind() { + let header_value: HeaderValue = true.into(); + assert_eq!(header_value.to_string_value(), "true"); } #[test] - fn value_only_to_string_for_int8_kind() { - let header_value = HeaderValue::from_int8(123).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123"); + fn to_string_value_for_int8_kind() { + let header_value: HeaderValue = 123i8.into(); + assert_eq!(header_value.to_string_value(), "123"); } #[test] - fn value_only_to_string_for_int16_kind() { - let header_value = HeaderValue::from_int16(12345).unwrap(); - assert_eq!(header_value.value_only_to_string(), "12345"); + fn to_string_value_for_int16_kind() { + let header_value: HeaderValue = 12345i16.into(); + assert_eq!(header_value.to_string_value(), "12345"); } #[test] - fn value_only_to_string_for_int32_kind() { - let header_value = HeaderValue::from_int32(123456).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123456"); + fn to_string_value_for_int32_kind() { + let header_value: HeaderValue = 123456i32.into(); + assert_eq!(header_value.to_string_value(), "123456"); } #[test] - fn value_only_to_string_for_int64_kind() { - let header_value = HeaderValue::from_int64(123456789).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123456789"); + fn to_string_value_for_int64_kind() { + let header_value: HeaderValue = 123456789i64.into(); + assert_eq!(header_value.to_string_value(), "123456789"); } #[test] - fn value_only_to_string_for_int128_kind() { - let header_value = HeaderValue::from_int128(123456789123456789).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123456789123456789"); + fn to_string_value_for_int128_kind() { + let header_value: HeaderValue = 123456789123456789i128.into(); + assert_eq!(header_value.to_string_value(), "123456789123456789"); } #[test] - fn value_only_to_string_for_uint8_kind() { - let header_value = HeaderValue::from_uint8(123).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123"); + fn to_string_value_for_uint8_kind() { + let header_value: HeaderValue = 123u8.into(); + assert_eq!(header_value.to_string_value(), "123"); } #[test] - fn value_only_to_string_for_uint16_kind() { - let header_value = HeaderValue::from_uint16(12345).unwrap(); - assert_eq!(header_value.value_only_to_string(), "12345"); + fn to_string_value_for_uint16_kind() { + let header_value: HeaderValue = 12345u16.into(); + assert_eq!(header_value.to_string_value(), "12345"); } #[test] - fn value_only_to_string_for_uint32_kind() { - let header_value = HeaderValue::from_uint32(123456).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123456"); + fn to_string_value_for_uint32_kind() { + let header_value: HeaderValue = 123456u32.into(); + assert_eq!(header_value.to_string_value(), "123456"); } #[test] - fn value_only_to_string_for_uint64_kind() { - let header_value = HeaderValue::from_uint64(123456789).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123456789"); + fn to_string_value_for_uint64_kind() { + let header_value: HeaderValue = 123456789u64.into(); + assert_eq!(header_value.to_string_value(), "123456789"); } #[test] - fn value_only_to_string_for_uint128_kind() { - let header_value = HeaderValue::from_uint128(123456789123456789).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123456789123456789"); + fn to_string_value_for_uint128_kind() { + let header_value: HeaderValue = 123456789123456789u128.into(); + assert_eq!(header_value.to_string_value(), "123456789123456789"); } #[test] - fn value_only_to_string_for_float32_kind() { - let header_value = HeaderValue::from_float32(123.01).unwrap(); - assert_eq!(header_value.value_only_to_string(), "123.01"); + fn to_string_value_for_float32_kind() { + let header_value: HeaderValue = 123.01f32.into(); + assert_eq!(header_value.to_string_value(), "123.01"); } #[test] - fn value_only_to_string_for_float64_kind() { - let header_value = HeaderValue::from_float64(1234.01234).unwrap(); - assert_eq!(header_value.value_only_to_string(), "1234.01234"); + fn to_string_value_for_float64_kind() { + let header_value: HeaderValue = 1234.01234f64.into(); + assert_eq!(header_value.to_string_value(), "1234.01234"); } #[test] fn should_be_serialized_as_bytes() { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("key-1").unwrap(), + HeaderKey::try_from("key-1").unwrap(), HeaderValue::from_str("Value 1").unwrap(), ); - headers.insert( - HeaderKey::new("key 1").unwrap(), - HeaderValue::from_uint64(12345).unwrap(), - ); - headers.insert( - HeaderKey::new("key_3").unwrap(), - HeaderValue::from_bool(true).unwrap(), - ); + headers.insert(HeaderKey::try_from("key 1").unwrap(), 12345u64.into()); + headers.insert(HeaderKey::try_from("key_3").unwrap(), true.into()); let bytes = headers.to_bytes(); let mut position = 0; let mut headers_count = 0; while position < bytes.len() { + let key_kind = HeaderKind::from_code(bytes[position]).unwrap(); + position += 1; let key_length = u32::from_le_bytes(bytes[position..position + 4].try_into().unwrap()) as usize; position += 4; - let key = String::from_utf8(bytes[position..position + key_length].to_vec()).unwrap(); + let key_value = bytes[position..position + key_length].to_vec(); position += key_length; - let kind = HeaderKind::from_code(bytes[position]).unwrap(); + + let value_kind = HeaderKind::from_code(bytes[position]).unwrap(); position += 1; let value_length = u32::from_le_bytes(bytes[position..position + 4].try_into().unwrap()) as usize; position += 4; let value = bytes[position..position + value_length].to_vec(); position += value_length; - let header = headers.get(&HeaderKey::new(&key).unwrap()); + + let key = HeaderKey { + kind: key_kind, + value: Bytes::from(key_value), + _marker: PhantomData, + }; + let header = headers.get(&key); assert!(header.is_some()); let header = header.unwrap(); - assert_eq!(header.kind, kind); + assert_eq!(header.kind, value_kind); assert_eq!(header.value, value); headers_count += 1; } @@ -1264,22 +1411,17 @@ mod tests { fn should_be_deserialized_from_bytes() { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("key-1").unwrap(), + HeaderKey::try_from("key-1").unwrap(), HeaderValue::from_str("Value 1").unwrap(), ); - headers.insert( - HeaderKey::new("key 2").unwrap(), - HeaderValue::from_uint64(12345).unwrap(), - ); - headers.insert( - HeaderKey::new("key_3").unwrap(), - HeaderValue::from_bool(true).unwrap(), - ); + headers.insert(HeaderKey::try_from("key 2").unwrap(), 12345u64.into()); + headers.insert(HeaderKey::try_from("key_3").unwrap(), true.into()); let mut bytes = BytesMut::new(); for (key, value) in &headers { - bytes.put_u32_le(key.0.len() as u32); - bytes.put_slice(key.0.as_bytes()); + bytes.put_u8(key.kind.as_code()); + bytes.put_u32_le(key.value.len() as u32); + bytes.put_slice(&key.value); bytes.put_u8(value.kind.as_code()); bytes.put_u32_le(value.value.len() as u32); bytes.put_slice(&value.value); @@ -1299,4 +1441,279 @@ mod tests { assert_eq!(deserialized_value.value, value.value); } } + + #[test] + fn should_serialize_and_deserialize_typed_keys() { + let mut headers = HashMap::new(); + headers.insert( + 123i32.into(), + HeaderValue::from_str("Value for int key").unwrap(), + ); + headers.insert(999u64.into(), true.into()); + + let bytes = headers.to_bytes(); + let deserialized = HashMap::::from_bytes(bytes).unwrap(); + + assert_eq!(deserialized.len(), headers.len()); + for (key, value) in &headers { + let deserialized_value = deserialized.get(key); + assert!(deserialized_value.is_some()); + let deserialized_value = deserialized_value.unwrap(); + assert_eq!(deserialized_value.kind, value.kind); + assert_eq!(deserialized_value.value, value.value); + } + } + + #[test] + fn header_value_should_be_created_from_vec_u8() { + let value = vec![1u8, 2, 3, 4, 5]; + let header_value = HeaderValue::try_from(value.clone()); + assert!(header_value.is_ok()); + let header_value = header_value.unwrap(); + assert_eq!(header_value.kind, HeaderKind::Raw); + assert_eq!(header_value.value.as_ref(), value.as_slice()); + assert_eq!(header_value.as_raw().unwrap(), value.as_slice()); + } + + #[test] + fn header_value_should_not_be_created_from_empty_vec_u8() { + let value: Vec = vec![]; + let header_value = HeaderValue::try_from(value); + assert!(header_value.is_err()); + let error = header_value.unwrap_err(); + assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); + } + + #[test] + fn header_value_should_not_be_created_from_too_long_vec_u8() { + let value: Vec = vec![0u8; 256]; + let header_value = HeaderValue::try_from(value); + assert!(header_value.is_err()); + let error = header_value.unwrap_err(); + assert_eq!(error.as_code(), IggyError::InvalidHeaderValue.as_code()); + } + + #[test] + fn header_key_should_be_created_from_vec_u8() { + let value = vec![1u8, 2, 3, 4]; + let header_key = HeaderKey::try_from(value.clone()); + assert!(header_key.is_ok()); + let header_key = header_key.unwrap(); + assert_eq!(header_key.kind, HeaderKind::Raw); + assert_eq!(header_key.value.as_ref(), value.as_slice()); + } + + #[test] + fn header_value_should_convert_to_bool() { + let header_value: HeaderValue = true.into(); + let extracted: bool = header_value.try_into().unwrap(); + assert!(extracted); + } + + #[test] + fn header_value_should_convert_to_i8() { + let header_value: HeaderValue = 42i8.into(); + let extracted: i8 = header_value.try_into().unwrap(); + assert_eq!(extracted, 42); + } + + #[test] + fn header_value_should_convert_to_i16() { + let header_value: HeaderValue = 1234i16.into(); + let extracted: i16 = header_value.try_into().unwrap(); + assert_eq!(extracted, 1234); + } + + #[test] + fn header_value_should_convert_to_i32() { + let header_value: HeaderValue = 123456i32.into(); + let extracted: i32 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123456); + } + + #[test] + fn header_value_should_convert_to_i64() { + let header_value: HeaderValue = 123456789i64.into(); + let extracted: i64 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123456789); + } + + #[test] + fn header_value_should_convert_to_i128() { + let header_value: HeaderValue = 123456789123456789i128.into(); + let extracted: i128 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123456789123456789); + } + + #[test] + fn header_value_should_convert_to_u8() { + let header_value: HeaderValue = 42u8.into(); + let extracted: u8 = header_value.try_into().unwrap(); + assert_eq!(extracted, 42); + } + + #[test] + fn header_value_should_convert_to_u16() { + let header_value: HeaderValue = 1234u16.into(); + let extracted: u16 = header_value.try_into().unwrap(); + assert_eq!(extracted, 1234); + } + + #[test] + fn header_value_should_convert_to_u32() { + let header_value: HeaderValue = 123456u32.into(); + let extracted: u32 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123456); + } + + #[test] + fn header_value_should_convert_to_u64() { + let header_value: HeaderValue = 123456789u64.into(); + let extracted: u64 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123456789); + } + + #[test] + fn header_value_should_convert_to_u128() { + let header_value: HeaderValue = 123456789123456789u128.into(); + let extracted: u128 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123456789123456789); + } + + #[test] + fn header_value_should_convert_to_f32() { + let header_value: HeaderValue = 123.5f32.into(); + let extracted: f32 = header_value.try_into().unwrap(); + assert_eq!(extracted, 123.5); + } + + #[test] + fn header_value_should_convert_to_f64() { + let header_value: HeaderValue = 1234.5678f64.into(); + let extracted: f64 = header_value.try_into().unwrap(); + assert_eq!(extracted, 1234.5678); + } + + #[test] + fn header_value_should_convert_to_string() { + let header_value = HeaderValue::try_from("hello").unwrap(); + let extracted: String = header_value.try_into().unwrap(); + assert_eq!(extracted, "hello"); + } + + #[test] + fn header_value_should_convert_to_vec_u8() { + let header_value = HeaderValue::try_from(vec![1u8, 2, 3]).unwrap(); + let extracted: Vec = header_value.try_into().unwrap(); + assert_eq!(extracted, vec![1u8, 2, 3]); + } + + #[test] + fn header_value_should_fail_conversion_with_wrong_kind() { + let header_value: HeaderValue = 42i32.into(); + let result: Result = header_value.try_into(); + assert!(result.is_err()); + } + + #[test] + fn header_value_ref_should_convert_to_i32() { + let value = 123456i32; + let header_value: HeaderValue = value.into(); + let extracted: i32 = (&header_value).try_into().unwrap(); + assert_eq!(extracted, value); + assert_eq!(header_value.as_int32().unwrap(), value); + } + + #[test] + fn to_string_value_handles_malformed_int32() { + let field = HeaderField:: { + kind: HeaderKind::Int32, + value: Bytes::from(vec![1, 2, 3]), // 3 bytes, needs 4 + _marker: PhantomData, + }; + assert_eq!(field.to_string_value(), ""); + } + + #[test] + fn as_bool_returns_error_on_empty_value() { + let field = HeaderField:: { + kind: HeaderKind::Bool, + value: Bytes::new(), + _marker: PhantomData, + }; + assert!(field.as_bool().is_err()); + } + + #[test] + fn as_uint8_returns_error_on_empty_value() { + let field = HeaderField:: { + kind: HeaderKind::Uint8, + value: Bytes::new(), + _marker: PhantomData, + }; + assert!(field.as_uint8().is_err()); + } + + #[test] + fn from_bytes_returns_error_on_truncated_input() { + let mut bytes = BytesMut::new(); + bytes.put_u8(2); // key_kind = String + bytes.put_u32_le(100); // key_len = 100 (lie!) + bytes.put_slice(b"abc"); // only 3 bytes of key data + + let result = HashMap::::from_bytes(bytes.freeze()); + assert!(result.is_err()); + } + + #[test] + fn display_handles_malformed_data() { + let field = HeaderField:: { + kind: HeaderKind::Float64, + value: Bytes::from(vec![1, 2]), // 2 bytes, needs 8 + _marker: PhantomData, + }; + let display = format!("{}", field); + assert!(display.contains("")); + } + + #[test] + fn from_bytes_returns_error_on_wrong_size_for_fixed_type() { + let mut bytes = BytesMut::new(); + // Key: Int32 with correct size + bytes.put_u8(6); // key_kind = Int32 + bytes.put_u32_le(4); // key_len = 4 (correct) + bytes.put_slice(&42i32.to_le_bytes()); + // Value: Int16 with wrong size + bytes.put_u8(5); // value_kind = Int16 + bytes.put_u32_le(4); // value_len = 4 (wrong, should be 2) + bytes.put_slice(&[1, 2, 3, 4]); + + let result = HashMap::::from_bytes(bytes.freeze()); + assert!(result.is_err()); + } + + #[test] + fn from_bytes_returns_error_on_truncated_value_length() { + let mut bytes = BytesMut::new(); + // Key: String + bytes.put_u8(2); // key_kind = String + bytes.put_u32_le(3); // key_len = 3 + bytes.put_slice(b"abc"); + // Value kind but no length bytes + bytes.put_u8(6); // value_kind = Int32 + // Missing value length bytes + + let result = HashMap::::from_bytes(bytes.freeze()); + assert!(result.is_err()); + } + + #[test] + fn to_string_value_handles_empty_bool() { + let field = HeaderField:: { + kind: HeaderKind::Bool, + value: Bytes::new(), + _marker: PhantomData, + }; + assert_eq!(field.to_string_value(), ""); + } } diff --git a/core/connectors/runtime/Cargo.toml b/core/connectors/runtime/Cargo.toml index 26c6fc9d32..270471abe4 100644 --- a/core/connectors/runtime/Cargo.toml +++ b/core/connectors/runtime/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy-connectors" -version = "0.2.1-edge.6" +version = "0.2.2-edge.1" description = "Connectors runtime for Iggy message streaming platform" edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sdk/Cargo.toml b/core/connectors/sdk/Cargo.toml index 530c5ba562..fced2f289c 100644 --- a/core/connectors/sdk/Cargo.toml +++ b/core/connectors/sdk/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_sdk" -version = "0.1.1-edge.3" +version = "0.1.2-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sinks/elasticsearch_sink/Cargo.toml b/core/connectors/sinks/elasticsearch_sink/Cargo.toml index 6d238eeca8..633effbf0f 100644 --- a/core/connectors/sinks/elasticsearch_sink/Cargo.toml +++ b/core/connectors/sinks/elasticsearch_sink/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_elasticsearch_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy Elasticsearch sink connector" edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sinks/iceberg_sink/Cargo.toml b/core/connectors/sinks/iceberg_sink/Cargo.toml index 8eb33b3450..4b63979de2 100644 --- a/core/connectors/sinks/iceberg_sink/Cargo.toml +++ b/core/connectors/sinks/iceberg_sink/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_iceberg_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" edition = "2024" license = "Apache-2.0" keywords = ["iggy", "messaging", "streaming"] diff --git a/core/connectors/sinks/postgres_sink/Cargo.toml b/core/connectors/sinks/postgres_sink/Cargo.toml index 4c9549830a..218a0635cf 100644 --- a/core/connectors/sinks/postgres_sink/Cargo.toml +++ b/core/connectors/sinks/postgres_sink/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_postgres_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy PostgreSQL sink connector for storing stream messages into PostgreSQL database" edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sinks/quickwit_sink/Cargo.toml b/core/connectors/sinks/quickwit_sink/Cargo.toml index 228b0e3bec..51fa78f100 100644 --- a/core/connectors/sinks/quickwit_sink/Cargo.toml +++ b/core/connectors/sinks/quickwit_sink/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_quickwit_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sinks/stdout_sink/Cargo.toml b/core/connectors/sinks/stdout_sink/Cargo.toml index ce63bc1ba6..bdceb6297e 100644 --- a/core/connectors/sinks/stdout_sink/Cargo.toml +++ b/core/connectors/sinks/stdout_sink/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_stdout_sink" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sources/elasticsearch_source/Cargo.toml b/core/connectors/sources/elasticsearch_source/Cargo.toml index e871ba3266..3f53386cbc 100644 --- a/core/connectors/sources/elasticsearch_source/Cargo.toml +++ b/core/connectors/sources/elasticsearch_source/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_elasticsearch_source" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy Elasticsearch source connector" edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sources/postgres_source/Cargo.toml b/core/connectors/sources/postgres_source/Cargo.toml index 6c82457079..694235390e 100644 --- a/core/connectors/sources/postgres_source/Cargo.toml +++ b/core/connectors/sources/postgres_source/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_postgres_source" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy PostgreSQL source connector supporting CDC and table polling for message streaming platform" edition = "2024" license = "Apache-2.0" diff --git a/core/connectors/sources/random_source/Cargo.toml b/core/connectors/sources/random_source/Cargo.toml index ba5acc6572..11b4d4a6ea 100644 --- a/core/connectors/sources/random_source/Cargo.toml +++ b/core/connectors/sources/random_source/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_connector_random_source" -version = "0.2.0-edge.1" +version = "0.2.1-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/integration/tests/cli/message/test_message_poll_command.rs b/core/integration/tests/cli/message/test_message_poll_command.rs index 2a4c2b4f38..7d75329129 100644 --- a/core/integration/tests/cli/message/test_message_poll_command.rs +++ b/core/integration/tests/cli/message/test_message_poll_command.rs @@ -195,9 +195,12 @@ impl IggyCmdTestCase for TestMessagePollCmd { if self.show_headers { status = status - .stdout(contains(format!("Header: {}", self.headers.0))) - .stdout(contains(self.headers.1.kind.to_string())) - .stdout(contains(self.headers.1.value_only_to_string()).count(self.message_count)) + .stdout(contains(format!( + "Header: {}", + self.headers.0.to_string_value() + ))) + .stdout(contains(self.headers.1.kind().to_string())) + .stdout(contains(self.headers.1.to_string_value()).count(self.message_count)) } // Check if messages are printed based on the strategy diff --git a/core/integration/tests/cli/message/test_message_send_command.rs b/core/integration/tests/cli/message/test_message_send_command.rs index 27227cd946..97ed4fe66c 100644 --- a/core/integration/tests/cli/message/test_message_send_command.rs +++ b/core/integration/tests/cli/message/test_message_send_command.rs @@ -109,7 +109,14 @@ impl TestMessageSendCmd { command.push( header .iter() - .map(|(k, v)| format!("{k}:{}:{}", v.kind, v.value_only_to_string())) + .map(|(k, v)| { + format!( + "{}:{}:{}", + k.to_string_value(), + v.kind(), + v.to_string_value() + ) + }) .collect::>() .join(","), ); @@ -340,13 +347,10 @@ pub async fn should_be_successful() { using_partitioning, Some(HashMap::from([ ( - HeaderKey::new("key1").unwrap(), - HeaderValue::from_kind_str_and_value_str("string", "value1").unwrap(), - ), - ( - HeaderKey::new("key2").unwrap(), - HeaderValue::from_kind_str_and_value_str("int32", "42").unwrap(), + HeaderKey::try_from("key1").unwrap(), + HeaderValue::try_from("value1").unwrap(), ), + (HeaderKey::try_from("key2").unwrap(), 42i32.into()), ])), )) .await; diff --git a/core/integration/tests/server/scenarios/create_message_payload.rs b/core/integration/tests/server/scenarios/create_message_payload.rs index a35a18afb8..1add0b0f75 100644 --- a/core/integration/tests/server/scenarios/create_message_payload.rs +++ b/core/integration/tests/server/scenarios/create_message_payload.rs @@ -20,7 +20,6 @@ use bytes::Bytes; use iggy::prelude::*; use integration::test_server::{ClientFactory, assert_clean_system, login_root}; use std::collections::HashMap; -use std::str::FromStr; const STREAM_NAME: &str = "test-stream"; const TOPIC_NAME: &str = "test-topic"; @@ -82,7 +81,7 @@ pub async fn run(client_factory: &dyn ClientFactory) { assert_eq!(headers.len(), 3); assert_eq!( headers - .get(&HeaderKey::new("key_1").unwrap()) + .get(&HeaderKey::try_from("key_1").unwrap()) .unwrap() .as_str() .unwrap(), @@ -90,14 +89,14 @@ pub async fn run(client_factory: &dyn ClientFactory) { ); assert!( headers - .get(&HeaderKey::new("key 2").unwrap()) + .get(&HeaderKey::try_from("key 2").unwrap()) .unwrap() .as_bool() .unwrap(), ); assert_eq!( headers - .get(&HeaderKey::new("key-3").unwrap()) + .get(&HeaderKey::try_from("key-3").unwrap()) .unwrap() .as_uint64() .unwrap(), @@ -141,16 +140,10 @@ fn create_message_payload(offset: u64) -> Bytes { fn create_message_headers() -> HashMap { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("key_1").unwrap(), - HeaderValue::from_str("Value 1").unwrap(), - ); - headers.insert( - HeaderKey::new("key 2").unwrap(), - HeaderValue::from_bool(true).unwrap(), - ); - headers.insert( - HeaderKey::new("key-3").unwrap(), - HeaderValue::from_uint64(123456).unwrap(), + HeaderKey::try_from("key_1").unwrap(), + HeaderValue::try_from("Value 1").unwrap(), ); + headers.insert(HeaderKey::try_from("key 2").unwrap(), true.into()); + headers.insert(HeaderKey::try_from("key-3").unwrap(), 123456u64.into()); headers } diff --git a/core/integration/tests/server/scenarios/encryption_scenario.rs b/core/integration/tests/server/scenarios/encryption_scenario.rs index 055f3ef46a..49f6081731 100644 --- a/core/integration/tests/server/scenarios/encryption_scenario.rs +++ b/core/integration/tests/server/scenarios/encryption_scenario.rs @@ -23,7 +23,7 @@ use integration::{ test_server::{ClientFactory, IpAddrKind, SYSTEM_PATH_ENV_VAR, TestServer, login_root}, }; use serial_test::parallel; -use std::{collections::HashMap, str::FromStr}; +use std::collections::HashMap; use test_case::test_matrix; fn encryption_enabled() -> bool { @@ -97,22 +97,13 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp for i in 0..messages_per_batch { let mut headers = HashMap::new(); + headers.insert(HeaderKey::try_from("batch").unwrap(), 1u64.into()); + headers.insert(HeaderKey::try_from("index").unwrap(), i.into()); headers.insert( - HeaderKey::new("batch").unwrap(), - HeaderValue::from_uint64(1).unwrap(), - ); - headers.insert( - HeaderKey::new("index").unwrap(), - HeaderValue::from_uint64(i).unwrap(), - ); - headers.insert( - HeaderKey::new("type").unwrap(), - HeaderValue::from_str("test-message").unwrap(), - ); - headers.insert( - HeaderKey::new("encrypted").unwrap(), - HeaderValue::from_bool(encryption).unwrap(), + HeaderKey::try_from("type").unwrap(), + HeaderValue::try_from("test-message").unwrap(), ); + headers.insert(HeaderKey::try_from("encrypted").unwrap(), encryption.into()); let message = IggyMessage::builder() .id((i + 1) as u128) @@ -195,7 +186,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp let headers = msg.user_headers_map().unwrap().unwrap(); assert_eq!( headers - .get(&HeaderKey::new("batch").unwrap()) + .get(&HeaderKey::try_from("batch").unwrap()) .unwrap() .as_uint64() .unwrap(), @@ -203,7 +194,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp ); assert_eq!( headers - .get(&HeaderKey::new("type").unwrap()) + .get(&HeaderKey::try_from("type").unwrap()) .unwrap() .as_str() .unwrap(), @@ -211,7 +202,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp ); assert_eq!( headers - .get(&HeaderKey::new("encrypted").unwrap()) + .get(&HeaderKey::try_from("encrypted").unwrap()) .unwrap() .as_bool() .unwrap(), @@ -242,22 +233,13 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp for i in 0..messages_per_batch { let mut headers = HashMap::new(); + headers.insert(HeaderKey::try_from("batch").unwrap(), 2u64.into()); + headers.insert(HeaderKey::try_from("index").unwrap(), i.into()); headers.insert( - HeaderKey::new("batch").unwrap(), - HeaderValue::from_uint64(2).unwrap(), - ); - headers.insert( - HeaderKey::new("index").unwrap(), - HeaderValue::from_uint64(i).unwrap(), - ); - headers.insert( - HeaderKey::new("type").unwrap(), - HeaderValue::from_str("test-message-after-restart").unwrap(), - ); - headers.insert( - HeaderKey::new("encrypted").unwrap(), - HeaderValue::from_bool(encryption).unwrap(), + HeaderKey::try_from("type").unwrap(), + HeaderValue::try_from("test-message-after-restart").unwrap(), ); + headers.insert(HeaderKey::try_from("encrypted").unwrap(), encryption.into()); let message = IggyMessage::builder() .id((messages_per_batch + i + 1) as u128) @@ -322,7 +304,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp assert!(msg.user_headers.is_some()); let headers = msg.user_headers_map().unwrap().unwrap(); let batch_num = headers - .get(&HeaderKey::new("batch").unwrap()) + .get(&HeaderKey::try_from("batch").unwrap()) .unwrap() .as_uint64() .unwrap(); @@ -331,7 +313,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp batch_1_count += 1; assert_eq!( headers - .get(&HeaderKey::new("type").unwrap()) + .get(&HeaderKey::try_from("type").unwrap()) .unwrap() .as_str() .unwrap(), @@ -339,7 +321,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp ); assert_eq!( headers - .get(&HeaderKey::new("encrypted").unwrap()) + .get(&HeaderKey::try_from("encrypted").unwrap()) .unwrap() .as_bool() .unwrap(), @@ -349,7 +331,7 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp batch_2_count += 1; assert_eq!( headers - .get(&HeaderKey::new("type").unwrap()) + .get(&HeaderKey::try_from("type").unwrap()) .unwrap() .as_str() .unwrap(), diff --git a/core/integration/tests/server/scenarios/message_headers_scenario.rs b/core/integration/tests/server/scenarios/message_headers_scenario.rs index 89c9aebee3..1d8b1e1905 100644 --- a/core/integration/tests/server/scenarios/message_headers_scenario.rs +++ b/core/integration/tests/server/scenarios/message_headers_scenario.rs @@ -23,7 +23,6 @@ use bytes::Bytes; use iggy::prelude::*; use integration::test_server::{ClientFactory, assert_clean_system, login_root}; use std::collections::HashMap; -use std::str::FromStr; pub async fn run(client_factory: &dyn ClientFactory) { let client = create_client(client_factory).await; @@ -79,7 +78,7 @@ pub async fn run(client_factory: &dyn ClientFactory) { assert_eq!(headers.len(), 3); assert_eq!( headers - .get(&HeaderKey::new("key_1").unwrap()) + .get(&HeaderKey::try_from("key_1").unwrap()) .unwrap() .as_str() .unwrap(), @@ -87,14 +86,14 @@ pub async fn run(client_factory: &dyn ClientFactory) { ); assert!( headers - .get(&HeaderKey::new("key 2").unwrap()) + .get(&HeaderKey::try_from("key 2").unwrap()) .unwrap() .as_bool() .unwrap(), ); assert_eq!( headers - .get(&HeaderKey::new("key-3").unwrap()) + .get(&HeaderKey::try_from("key-3").unwrap()) .unwrap() .as_uint64() .unwrap(), @@ -131,16 +130,10 @@ fn create_message_payload(offset: u64) -> Bytes { fn create_message_headers() -> HashMap { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("key_1").unwrap(), - HeaderValue::from_str("Value 1").unwrap(), - ); - headers.insert( - HeaderKey::new("key 2").unwrap(), - HeaderValue::from_bool(true).unwrap(), - ); - headers.insert( - HeaderKey::new("key-3").unwrap(), - HeaderValue::from_uint64(123456).unwrap(), + HeaderKey::try_from("key_1").unwrap(), + HeaderValue::try_from("Value 1").unwrap(), ); + headers.insert(HeaderKey::try_from("key 2").unwrap(), true.into()); + headers.insert(HeaderKey::try_from("key-3").unwrap(), 123456u64.into()); headers } diff --git a/core/integration/tests/server/scenarios/message_size_scenario.rs b/core/integration/tests/server/scenarios/message_size_scenario.rs index f2742c8fac..f63b0570bd 100644 --- a/core/integration/tests/server/scenarios/message_size_scenario.rs +++ b/core/integration/tests/server/scenarios/message_size_scenario.rs @@ -20,7 +20,6 @@ use bytes::Bytes; use iggy::prelude::*; use integration::test_server::{ClientFactory, assert_clean_system, login_root}; use std::collections::HashMap; -use std::str::FromStr; const STREAM_NAME: &str = "test-stream"; const TOPIC_NAME: &str = "test-topic"; @@ -202,29 +201,30 @@ fn create_string_of_size(size: usize) -> String { fn create_message_header_of_size(target_size: usize) -> HashMap { let mut headers = HashMap::new(); - let mut header_id = 1; let mut current_size = 0; + let mut header_id: u32 = 0; while current_size < target_size { let remaining_size = target_size - current_size; + let key_bytes = header_id.to_le_bytes(); + let key_len = key_bytes.len(); + let header_overhead = 1 + 4 + key_len + 1 + 4; + let min_header_size = header_overhead + 1; - let key_str = format!("header-{header_id}"); - let key_overhead = 4; // 4 bytes for key length - let value_overhead = 5; // 1 byte for type + 4 bytes for value length - let total_overhead = key_overhead + key_str.len() + value_overhead; - - let value_size = if remaining_size <= total_overhead { + if remaining_size < min_header_size { break; - } else if remaining_size - total_overhead > MAX_SINGLE_HEADER_SIZE { + } + + let value_size = if remaining_size - header_overhead > MAX_SINGLE_HEADER_SIZE { MAX_SINGLE_HEADER_SIZE } else { - remaining_size - total_overhead + remaining_size - header_overhead }; - let key = HeaderKey::new(key_str.as_str()).unwrap(); - let value = HeaderValue::from_str(create_string_of_size(value_size).as_str()).unwrap(); + let key = HeaderKey::try_from(key_bytes.as_slice()).unwrap(); + let value = HeaderValue::try_from(create_string_of_size(value_size).as_str()).unwrap(); - let actual_header_size = 4 + key_str.len() + 1 + 4 + value_size; + let actual_header_size = header_overhead + value_size; current_size += actual_header_size; headers.insert(key, value); diff --git a/core/integration/tests/server/scenarios/offset_scenario.rs b/core/integration/tests/server/scenarios/offset_scenario.rs index cc519bd634..608b1b3fb9 100644 --- a/core/integration/tests/server/scenarios/offset_scenario.rs +++ b/core/integration/tests/server/scenarios/offset_scenario.rs @@ -21,7 +21,6 @@ use bytes::BytesMut; use iggy::prelude::*; use integration::test_server::{ClientFactory, login_root}; use std::collections::HashMap; -use std::str::FromStr; fn small_batches() -> Vec { vec![3, 4, 5, 6, 7] @@ -174,17 +173,11 @@ fn create_single_message(id: u32, message_size: u64) -> IggyMessage { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("key_1").unwrap(), - HeaderValue::from_str("Value 1").unwrap(), - ); - headers.insert( - HeaderKey::new("key 2").unwrap(), - HeaderValue::from_bool(true).unwrap(), - ); - headers.insert( - HeaderKey::new("key-3").unwrap(), - HeaderValue::from_uint64(123456).unwrap(), + HeaderKey::try_from("key_1").unwrap(), + HeaderValue::try_from("Value 1").unwrap(), ); + headers.insert(HeaderKey::try_from("key 2").unwrap(), true.into()); + headers.insert(HeaderKey::try_from("key-3").unwrap(), 123456u64.into()); IggyMessage::builder() .id(id as u128) diff --git a/core/integration/tests/server/scenarios/timestamp_scenario.rs b/core/integration/tests/server/scenarios/timestamp_scenario.rs index 542ca17cc8..eb5129701f 100644 --- a/core/integration/tests/server/scenarios/timestamp_scenario.rs +++ b/core/integration/tests/server/scenarios/timestamp_scenario.rs @@ -21,7 +21,6 @@ use bytes::BytesMut; use iggy::prelude::*; use integration::test_server::{ClientFactory, login_root}; use std::collections::HashMap; -use std::str::FromStr; use tokio::time::{Duration, sleep}; fn small_batches() -> Vec { @@ -198,17 +197,11 @@ fn create_single_message(id: u32, message_size: u64) -> IggyMessage { let mut headers = HashMap::new(); headers.insert( - HeaderKey::new("key_1").unwrap(), - HeaderValue::from_str("Value 1").unwrap(), - ); - headers.insert( - HeaderKey::new("key 2").unwrap(), - HeaderValue::from_bool(true).unwrap(), - ); - headers.insert( - HeaderKey::new("key-3").unwrap(), - HeaderValue::from_uint64(123456).unwrap(), + HeaderKey::try_from("key_1").unwrap(), + HeaderValue::try_from("Value 1").unwrap(), ); + headers.insert(HeaderKey::try_from("key 2").unwrap(), true.into()); + headers.insert(HeaderKey::try_from("key-3").unwrap(), 123456u64.into()); IggyMessage::builder() .id(id as u128) diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml index 3f16531c67..4fa28e16db 100644 --- a/core/sdk/Cargo.toml +++ b/core/sdk/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy" -version = "0.8.1-edge.7" +version = "0.8.2-edge.1" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" license = "Apache-2.0" diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs index ba374fe6b7..bcbb44f827 100644 --- a/core/sdk/src/prelude.rs +++ b/core/sdk/src/prelude.rs @@ -57,12 +57,12 @@ pub use iggy_common::{ Aes256GcmEncryptor, Args, ArgsOptional, AutoLogin, BytesSerializable, CacheMetrics, CacheMetricsKey, ClientError, ClientInfoDetails, ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroupDetails, ConsumerKind, - EncryptorKind, FlushUnsavedBuffer, GlobalPermissions, HeaderKey, HeaderValue, HttpClientConfig, - HttpClientConfigBuilder, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, - IggyError, IggyExpiry, IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView, - IggyMessageView, IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, Partition, Partitioner, - Partitioning, Permissions, PersonalAccessTokenExpiry, PollMessages, PolledMessages, - PollingKind, PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, + EncryptorKind, FlushUnsavedBuffer, GlobalPermissions, HeaderKey, HeaderKind, HeaderValue, + HttpClientConfig, HttpClientConfigBuilder, IdKind, Identifier, IdentityInfo, IggyByteSize, + IggyDuration, IggyError, IggyExpiry, IggyIndexView, IggyMessage, IggyMessageHeader, + IggyMessageHeaderView, IggyMessageView, IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, + Partition, Partitioner, Partitioning, Permissions, PersonalAccessTokenExpiry, PollMessages, + PolledMessages, PollingKind, PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, SendMessages, Sizeable, SnapshotCompression, Stats, Stream, StreamDetails, StreamPermissions, SystemSnapshotType, TcpClientConfig, TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicDetails, TopicPermissions, TransportEndpoints, diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index 8adfda896b..c2644498df 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "server" -version = "0.6.1-edge.6" +version = "0.6.2-edge.1" edition = "2024" license = "Apache-2.0" diff --git a/core/server/server.http b/core/server/server.http index 02a7d2d137..a2dbabf9c9 100644 --- a/core/server/server.http +++ b/core/server/server.http @@ -25,7 +25,10 @@ @partition_id_payload_base64 = AAAAAA== @message_1_payload_base64 = aGVsbG8= @message_2_payload_base64 = d29ybGQ= -@header_1_payload_base_64 = dmFsdWUgMQ== +@header_key_1_base64 = a2V5XzE= +@header_value_1_base64 = dmFsdWUgMQ== +@header_key_2_base64 = Kg== +@header_value_2_base64 = AAAA @root_username = iggy @root_password = iggy @user1_username = user1 @@ -301,12 +304,25 @@ Content-Type: application/json }, { "id": 0, "payload": "{{message_2_payload_base64}}", - "headers": { - "key_1": { + "user_headers": [{ + "key": { "kind": "string", - "value": "{{header_1_payload_base_64}}" + "value": "{{header_key_1_base64}}" + }, + "value": { + "kind": "string", + "value": "{{header_value_1_base64}}" } - } + }, { + "key": { + "kind": "uint32", + "value": "{{header_key_2_base64}}" + }, + "value": { + "kind": "int32", + "value": "{{header_value_2_base64}}" + } + }] }] } diff --git a/core/tools/src/data-seeder/seeder.rs b/core/tools/src/data-seeder/seeder.rs index bbd176fbd0..edbac98dc3 100644 --- a/core/tools/src/data-seeder/seeder.rs +++ b/core/tools/src/data-seeder/seeder.rs @@ -19,7 +19,6 @@ use iggy::prelude::*; use rand::Rng; use std::collections::HashMap; -use std::str::FromStr; const PROD_STREAM_NAME: &str = "prod"; const TEST_STREAM_NAME: &str = "test"; @@ -157,13 +156,12 @@ async fn send_messages(client: &IggyClient, streams: &[(String, u32)]) -> Result false => None, true => { let mut headers = HashMap::new(); - headers - .insert(HeaderKey::new("key 1")?, HeaderValue::from_str("value1")?); - headers.insert(HeaderKey::new("key-2")?, HeaderValue::from_bool(true)?); headers.insert( - HeaderKey::new("key_3")?, - HeaderValue::from_uint64(123456)?, + HeaderKey::try_from("key 1")?, + HeaderValue::try_from("value1")?, ); + headers.insert(HeaderKey::try_from("key-2")?, true.into()); + headers.insert(HeaderKey::try_from("key_3")?, 123456u64.into()); Some(headers) } }; diff --git a/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Consumer/Utils.cs b/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Consumer/Utils.cs index 0fcfef20d5..4870089cc5 100644 --- a/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Consumer/Utils.cs +++ b/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Consumer/Utils.cs @@ -89,7 +89,7 @@ public static async Task ConsumeMessages(IIggyClient client, ILogger logger) private static void HandleMessage(MessageResponse message, ILogger logger) { - var headerKey = HeaderKey.New("message_type"); + var headerKey = HeaderKey.FromString("message_type"); var headersMap = message.UserHeaders ?? throw new Exception("Missing headers map."); var messageType = Encoding.UTF8.GetString(headersMap[headerKey].Value); diff --git a/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Producer/Utils.cs b/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Producer/Utils.cs index e387007622..cb7b60f6ee 100644 --- a/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Producer/Utils.cs +++ b/examples/csharp/src/MessageHeaders/Iggy_SDK.Examples.MessageHeaders.Producer/Utils.cs @@ -106,7 +106,7 @@ public static async Task ProduceMessages(IIggyClient client, ILogger logger) return new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(jsonEnvelope), new Dictionary { - { HeaderKey.New("message_type"), HeaderValue.FromString(serializableMessage.MessageType) } + { HeaderKey.FromString("message_type"), HeaderValue.FromString(serializableMessage.MessageType) } }); } ).ToList(); diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/consumer/MessageHeadersConsumer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/consumer/MessageHeadersConsumer.java index 346e2fcdbe..a5ea7e00ec 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/consumer/MessageHeadersConsumer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/consumer/MessageHeadersConsumer.java @@ -27,6 +27,7 @@ import org.apache.iggy.examples.shared.Messages.OrderRejected; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.Message; import org.apache.iggy.message.PolledMessages; @@ -42,6 +43,7 @@ import java.util.Optional; public final class MessageHeadersConsumer { + private static final String STREAM_NAME = "headers-stream"; private static final StreamId STREAM_ID = StreamId.of(STREAM_NAME); @@ -110,7 +112,6 @@ private static void consumeMessages(IggyTcpClient client) { consumedBatches++; offset = offset.add(BigInteger.valueOf(polledMessages.messages().size())); - } catch (Exception e) { log.error("Error polling messages", e); break; @@ -123,13 +124,13 @@ private static void handleMessage(Message message) { String messageType = "unknown"; try { - Map userHeaders = message.userHeaders(); + Map userHeaders = message.userHeaders(); if (userHeaders.isEmpty()) { log.warn("Missing headers at offset {}.", message.header().offset()); return; } - HeaderValue headerValue = userHeaders.get(MESSAGE_TYPE_HEADER); + HeaderValue headerValue = userHeaders.get(HeaderKey.fromString(MESSAGE_TYPE_HEADER)); if (headerValue == null) { log.warn( "Missing message type header at offset {}.", @@ -137,7 +138,7 @@ private static void handleMessage(Message message) { return; } - messageType = headerValue.value(); + messageType = headerValue.asString(); log.info( "Handling message type: {} at offset: {}...", messageType, diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java index c703a537df..530b9dc9e9 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java @@ -24,7 +24,7 @@ import org.apache.iggy.examples.shared.MessagesGenerator; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; -import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; @@ -42,6 +42,7 @@ import java.util.Optional; public final class MessageHeadersProducer { + private static final String STREAM_NAME = "headers-stream"; private static final StreamId STREAM_ID = StreamId.of(STREAM_NAME); @@ -111,8 +112,8 @@ public static void produceMessages(IggyTcpClient client) { SerializableMessage serializableMessage = generator.generate(); String messageType = serializableMessage.getMessageType(); String json = serializableMessage.toJson(); - Map userHeaders = new HashMap<>(); - userHeaders.put(MESSAGE_TYPE_HEADER, new HeaderValue(HeaderKind.String, messageType)); + Map userHeaders = new HashMap<>(); + userHeaders.put(HeaderKey.fromString(MESSAGE_TYPE_HEADER), HeaderValue.fromString(messageType)); messages.add(Message.of(json, userHeaders)); serializableMessages.add(serializableMessage); } diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml index 6fd30e6b5a..0dc7694ac5 100644 --- a/examples/rust/Cargo.toml +++ b/examples/rust/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "iggy_examples" -version = "0.0.5" +version = "0.0.6" edition = "2024" license = "Apache-2.0" @@ -78,6 +78,14 @@ path = "src/message-headers/message-compression/consumer/main.rs" name = "message-headers-compression-producer" path = "src/message-headers/message-compression/producer/main.rs" +[[example]] +name = "typed-headers-consumer" +path = "src/message-headers/typed-headers/consumer/main.rs" + +[[example]] +name = "typed-headers-producer" +path = "src/message-headers/typed-headers/producer/main.rs" + [[example]] name = "multi-tenant-consumer" path = "src/multi-tenant/consumer/main.rs" diff --git a/examples/rust/README.md b/examples/rust/README.md index 0f55671cc3..4a546f73d9 100644 --- a/examples/rust/README.md +++ b/examples/rust/README.md @@ -101,6 +101,8 @@ cargo run --example message-headers-type-producer cargo run --example message-headers-type-consumer ``` +Demonstrates using HeaderKey/HeaderValue for message metadata instead of payload-based typing, with header-based message routing. + Shows how user headers can be used for message compression in transit: ```bash @@ -108,7 +110,12 @@ cargo run --example message-headers-compression-producer cargo run --example message-headers-compression-consumer ``` -Demonstrates using HeaderKey/HeaderValue for message metadata instead of payload-based typing, with header-based message routing. +Demonstrates typed header keys and values with various data types (strings, integers, floats, booleans, raw bytes): + +```bash +cargo run --example typed-headers-producer +cargo run --example typed-headers-consumer +``` ### Message Envelopes diff --git a/examples/rust/src/message-headers/message-type/consumer/main.rs b/examples/rust/src/message-headers/message-type/consumer/main.rs index d3ad3ca64b..8ac62ffbfa 100644 --- a/examples/rust/src/message-headers/message-type/consumer/main.rs +++ b/examples/rust/src/message-headers/message-type/consumer/main.rs @@ -51,7 +51,7 @@ fn handle_message(message: &IggyMessage) -> Result<(), Box> { // The payload can be of any type as it is a raw byte array. In this case it's a JSON string. let payload = std::str::from_utf8(&message.payload)?; // The message type is stored in the custom message header. - let header_key = HeaderKey::new("message_type").unwrap(); + let header_key = HeaderKey::try_from("message_type").unwrap(); let headers_map = message.user_headers_map()?.unwrap(); let message_type = headers_map.get(&header_key).unwrap().as_str()?; info!( diff --git a/examples/rust/src/message-headers/message-type/producer/main.rs b/examples/rust/src/message-headers/message-type/producer/main.rs index 4b762c32cf..f394fddfeb 100644 --- a/examples/rust/src/message-headers/message-type/producer/main.rs +++ b/examples/rust/src/message-headers/message-type/producer/main.rs @@ -24,7 +24,6 @@ use iggy_examples::shared::messages_generator::MessagesGenerator; use iggy_examples::shared::system; use std::collections::HashMap; use std::error::Error; -use std::str::FromStr; use std::sync::Arc; use tracing::info; use tracing_subscriber::layer::SubscriberExt; @@ -86,8 +85,8 @@ async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box Result<(), Box> { + let args = Args::parse_with_defaults("typed-headers-consumer"); + Registry::default() + .with(tracing_subscriber::fmt::layer()) + .with(EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO"))) + .init(); + info!( + "Typed headers consumer has started, selected transport: {}", + args.transport + ); + let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); + let client = client_provider::get_raw_client(client_provider_config, false).await?; + let client = IggyClient::new(client); + client.connect().await?; + system::init_by_consumer(&args, &client).await; + system::consume_messages(&args, &client, &handle_message).await +} + +fn handle_message(message: &IggyMessage) -> Result<(), Box> { + let payload = std::str::from_utf8(&message.payload)?; + + info!( + "Message at offset: {}, payload: {}", + message.header.offset, payload + ); + + if let Some(headers_map) = message.user_headers_map()? { + info!("Headers ({}):", headers_map.len()); + for (key, value) in &headers_map { + info!( + " key: [kind={}, value={}] -> value: [kind={}, value={}]", + key.kind(), + key.to_string_value(), + value.kind(), + value.to_string_value() + ); + } + } + + Ok(()) +} diff --git a/examples/rust/src/message-headers/typed-headers/producer/main.rs b/examples/rust/src/message-headers/typed-headers/producer/main.rs new file mode 100644 index 0000000000..3e6d3c5a88 --- /dev/null +++ b/examples/rust/src/message-headers/typed-headers/producer/main.rs @@ -0,0 +1,120 @@ +/* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use anyhow::Result; +use bytes::Bytes; +use iggy::prelude::*; +use iggy_examples::shared::args::Args; +use iggy_examples::shared::system; +use std::collections::HashMap; +use std::error::Error; +use std::sync::Arc; +use tracing::info; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, Registry}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::parse_with_defaults("typed-headers-producer"); + Registry::default() + .with(tracing_subscriber::fmt::layer()) + .with(EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO"))) + .init(); + info!( + "Typed headers producer has started, selected transport: {}", + args.transport + ); + let client_provider_config = Arc::new(ClientProviderConfig::from_args(args.to_sdk_args())?); + let client = client_provider::get_raw_client(client_provider_config, false).await?; + let client = IggyClient::new(client); + client.connect().await?; + system::init_by_producer(&args, &client).await?; + produce_messages(&args, &client).await +} + +async fn produce_messages(args: &Args, client: &IggyClient) -> Result<(), Box> { + let interval = args.get_interval(); + info!( + "Messages will be sent to stream: {}, topic: {}, partition: {} with interval {}.", + args.stream_id, + args.topic_id, + args.partition_id, + interval.map_or("none".to_string(), |i| i.as_human_time_string()) + ); + let stream_id = args.stream_id.clone().try_into()?; + let topic_id = args.topic_id.clone().try_into()?; + let mut interval = interval.map(|interval| tokio::time::interval(interval.get_duration())); + let mut sent_batches = 0; + let mut message_id: u64 = 0; + let partitioning = Partitioning::partition_id(args.partition_id); + + loop { + if args.message_batches_limit > 0 && sent_batches == args.message_batches_limit { + info!("Sent {sent_batches} batches of messages, exiting."); + return Ok(()); + } + + if let Some(interval) = &mut interval { + interval.tick().await; + } + + let mut messages = Vec::new(); + for _ in 0..args.messages_per_batch { + message_id += 1; + + let mut headers = HashMap::new(); + headers.insert( + HeaderKey::try_from("event_type")?, + HeaderValue::try_from("user_action")?, + ); + headers.insert(1u32.into(), message_id.into()); + headers.insert( + HeaderKey::try_from("important")?, + message_id.is_multiple_of(5).into(), + ); + headers.insert(44u32.into(), (message_id as f64 * 2.0).into()); + headers.insert( + HeaderKey::try_from("trace_id")?, + (message_id as i128 * 1_000_000_000_000).into(), + ); + headers.insert( + HeaderKey::try_from([0xDE, 0xAD].as_slice())?, + HeaderValue::try_from([0xBE, 0xEF, 0xCA, 0xFE].as_slice())?, + ); + + let payload = + format!(r#"{{"message_id":{message_id},"content":"Hello from typed headers!"}}"#,); + + let message = IggyMessage::builder() + .payload(Bytes::from(payload)) + .user_headers(headers) + .build()?; + messages.push(message); + } + + client + .send_messages(&stream_id, &topic_id, &partitioning, &mut messages) + .await?; + sent_batches += 1; + info!( + "Sent batch {} with {} messages (last message_id: {})", + sent_batches, args.messages_per_batch, message_id + ); + } +} diff --git a/examples/rust/src/shared/codec.rs b/examples/rust/src/shared/codec.rs index 30df429be8..3625be4174 100644 --- a/examples/rust/src/shared/codec.rs +++ b/examples/rust/src/shared/codec.rs @@ -57,13 +57,13 @@ impl FromStr for Codec { impl Codec { /// Returns the key to indicate compressed messages as HeaderKey. pub fn header_key() -> HeaderKey { - HeaderKey::new(COMPRESSION_HEADER_KEY) + HeaderKey::try_from(COMPRESSION_HEADER_KEY) .expect("COMPRESSION_HEADER_KEY is an InvalidHeaderKey.") } /// Returns the compression algorithm type as HeaderValue. pub fn to_header_value(&self) -> HeaderValue { - HeaderValue::from_str(&self.to_string()).expect("failed generating HeaderValue.") + HeaderValue::try_from(self.to_string()).expect("failed generating HeaderValue.") } /// Returns a Codec from a HeaderValue. Used when reading messages from the server. diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs index d3e2750c1b..b5d0fae1e4 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs @@ -105,8 +105,8 @@ public async Task PollMessages_WithHeaders_Should_PollMessages_Successfully(Prot { responseMessage.UserHeaders.ShouldNotBeNull(); responseMessage.UserHeaders.Count.ShouldBe(2); - responseMessage.UserHeaders[HeaderKey.New("header1")].ToString().ShouldBe("value1"); - responseMessage.UserHeaders[HeaderKey.New("header2")].ToInt32().ShouldBeGreaterThan(0); + responseMessage.UserHeaders[HeaderKey.FromString("header1")].ToString().ShouldBe("value1"); + responseMessage.UserHeaders[HeaderKey.FromString("header2")].ToInt32().ShouldBeGreaterThan(0); } } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/FetchMessagesFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/FetchMessagesFixture.cs index c8ec18148b..e58e2742fa 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/FetchMessagesFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/FetchMessagesFixture.cs @@ -95,8 +95,8 @@ private static Message[] CreateMessagesWithHeader(int count) messages.Add(new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), new Dictionary { - { HeaderKey.New("header1"), HeaderValue.FromString("value1") }, - { HeaderKey.New("header2"), HeaderValue.FromInt32(14 + i) } + { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, + { HeaderKey.FromString("header2"), HeaderValue.FromInt32(14 + i) } })); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs index af465a28de..3973415d07 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs @@ -59,14 +59,14 @@ public static Task Before() new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), new Dictionary { - { HeaderKey.New("header1"), HeaderValue.FromString("value1") }, - { HeaderKey.New("header2"), HeaderValue.FromInt32(444) } + { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, + { HeaderKey.FromString("header2"), HeaderValue.FromInt32(444) } }), new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes(dummyJson), new Dictionary { - { HeaderKey.New("header1"), HeaderValue.FromString("value1") }, - { HeaderKey.New("header2"), HeaderValue.FromInt32(444) } + { HeaderKey.FromString("header1"), HeaderValue.FromString("value1") }, + { HeaderKey.FromString("header2"), HeaderValue.FromInt32(444) } }) ]; diff --git a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs index af47091ab1..f39ddd8645 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/MessageResponse.cs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. +using System.Text.Json.Serialization; using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; using Apache.Iggy.Messages; namespace Apache.Iggy.Contracts; @@ -38,5 +40,7 @@ public sealed class MessageResponse /// /// Headers defined by the user. /// + [JsonPropertyName("user_headers")] + [JsonConverter(typeof(UserHeadersConverter))] public Dictionary? UserHeaders { get; init; } } diff --git a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs index 7f576c891e..f2753e51ca 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs @@ -554,12 +554,12 @@ private static byte[] GetHeadersBytes(Dictionary? header return []; } - var headersLength = headers.Sum(header => 4 + header.Key.Value.Length + 1 + 4 + header.Value.Value.Length); + var headersLength = headers.Sum(kvp => 1 + 4 + kvp.Key.Value.Length + 1 + 4 + kvp.Value.Value.Length); Span headersBytes = stackalloc byte[headersLength]; var position = 0; - foreach (var (headerKey, headerValue) in headers) + foreach (var kvp in headers) { - var headerBytes = GetBytesFromHeader(headerKey, headerValue); + var headerBytes = GetBytesFromHeader(kvp.Key, kvp.Value); headerBytes.CopyTo(headersBytes[position..(position + headerBytes.Length)]); position += headerBytes.Length; } @@ -575,9 +575,13 @@ private static byte HeaderKindToByte(HeaderKind kind) HeaderKind.Raw => 1, HeaderKind.String => 2, HeaderKind.Bool => 3, + HeaderKind.Int8 => 4, + HeaderKind.Int16 => 5, HeaderKind.Int32 => 6, HeaderKind.Int64 => 7, HeaderKind.Int128 => 8, + HeaderKind.Uint8 => 9, + HeaderKind.Uint16 => 10, HeaderKind.Uint32 => 11, HeaderKind.Uint64 => 12, HeaderKind.Uint128 => 13, @@ -589,19 +593,22 @@ private static byte HeaderKindToByte(HeaderKind kind) private static byte[] GetBytesFromHeader(HeaderKey headerKey, HeaderValue headerValue) { - var headerBytesLength = 4 + headerKey.Value.Length + 1 + 4 + headerValue.Value.Length; + var headerBytesLength = 1 + 4 + headerKey.Value.Length + 1 + 4 + headerValue.Value.Length; Span headerBytes = stackalloc byte[headerBytesLength]; + var pos = 0; - BinaryPrimitives.WriteInt32LittleEndian(headerBytes[..4], headerKey.Value.Length); - var headerKeyBytes = Encoding.UTF8.GetBytes(headerKey.Value); - headerKeyBytes.CopyTo(headerBytes[4..(4 + headerKey.Value.Length)]); + headerBytes[pos++] = HeaderKindToByte(headerKey.Kind); - headerBytes[4 + headerKey.Value.Length] = HeaderKindToByte(headerValue.Kind); + BinaryPrimitives.WriteInt32LittleEndian(headerBytes[pos..(pos + 4)], headerKey.Value.Length); + pos += 4; + headerKey.Value.CopyTo(headerBytes[pos..(pos + headerKey.Value.Length)]); + pos += headerKey.Value.Length; - BinaryPrimitives.WriteInt32LittleEndian( - headerBytes[(4 + headerKey.Value.Length + 1)..(4 + headerKey.Value.Length + 1 + 4)], - headerValue.Value.Length); - headerValue.Value.CopyTo(headerBytes[(4 + headerKey.Value.Length + 1 + 4)..]); + headerBytes[pos++] = HeaderKindToByte(headerValue.Kind); + + BinaryPrimitives.WriteInt32LittleEndian(headerBytes[pos..(pos + 4)], headerValue.Value.Length); + pos += 4; + headerValue.Value.CopyTo(headerBytes[pos..]); return headerBytes.ToArray(); } diff --git a/foreign/csharp/Iggy_SDK/Headers/HeaderKey.cs b/foreign/csharp/Iggy_SDK/Headers/HeaderKey.cs index 084632929e..b6a19ad213 100644 --- a/foreign/csharp/Iggy_SDK/Headers/HeaderKey.cs +++ b/foreign/csharp/Iggy_SDK/Headers/HeaderKey.cs @@ -15,48 +15,70 @@ // specific language governing permissions and limitations // under the License. -using System.Text.Json.Serialization; -using Apache.Iggy.JsonConverters; +using System.Text; namespace Apache.Iggy.Headers; /// -/// A key for a header. +/// Represents a message header key with a kind and binary value. /// -[JsonConverter(typeof(HeaderKeyConverter))] public readonly struct HeaderKey : IEquatable { /// - /// Header key value. + /// The kind of the header key. /// - public required string Value { get; init; } + public required HeaderKind Kind { get; init; } /// - /// Creates a new header key from a string. + /// The binary value of the header key. /// - /// Key value - /// - /// - public static HeaderKey New(string val) + public required byte[] Value { get; init; } + + /// + /// Creates a HeaderKey from a string value. + /// + /// The string value (must be 1-255 characters). + /// A new HeaderKey with String kind. + /// Thrown when value length is invalid. + public static HeaderKey FromString(string val) { + if (val.Length is 0 or > 255) + { + throw new ArgumentException("Value has incorrect size, must be between 1 and 255", nameof(val)); + } + return new HeaderKey { - Value = val.Length is 0 or > 255 - ? throw new ArgumentException("Value has incorrect size, must be between 1 and 255", nameof(val)) - : val + Kind = HeaderKind.String, + Value = Encoding.UTF8.GetBytes(val) }; } + /// + /// Gets the value as a UTF-8 string. + /// + /// The string value. + /// Thrown when kind is not String. + public string AsString() + { + if (Kind is not HeaderKind.String) + { + throw new InvalidOperationException("HeaderKey is not of String kind"); + } + + return Encoding.UTF8.GetString(Value); + } + /// public override string ToString() { - return Value; + return Kind == HeaderKind.String ? Encoding.UTF8.GetString(Value) : Convert.ToBase64String(Value); } /// public bool Equals(HeaderKey other) { - return StringComparer.Ordinal.Equals(Value, other.Value); + return Kind == other.Kind && Value.SequenceEqual(other.Value); } /// @@ -68,26 +90,27 @@ public override bool Equals(object? obj) /// public override int GetHashCode() { - return StringComparer.Ordinal.GetHashCode(Value); + var hash = new HashCode(); + hash.Add(Kind); + foreach (var b in Value) + { + hash.Add(b); + } + + return hash.ToHashCode(); } /// - /// Determines whether two specified objects are equal. + /// Determines whether two HeaderKey instances are equal. /// - /// The first to compare. - /// The second to compare. - /// True if the two objects are equal; otherwise, false. public static bool operator ==(HeaderKey left, HeaderKey right) { return left.Equals(right); } /// - /// Determines whether two specified objects are not equal. + /// Determines whether two HeaderKey instances are not equal. /// - /// The first to compare. - /// The second to compare. - /// True if the two objects are not equal; otherwise, false. public static bool operator !=(HeaderKey left, HeaderKey right) { return !left.Equals(right); diff --git a/foreign/csharp/Iggy_SDK/Headers/HeaderKind.cs b/foreign/csharp/Iggy_SDK/Headers/HeaderKind.cs index 493a4abe91..fe2d3436c3 100644 --- a/foreign/csharp/Iggy_SDK/Headers/HeaderKind.cs +++ b/foreign/csharp/Iggy_SDK/Headers/HeaderKind.cs @@ -38,7 +38,17 @@ public enum HeaderKind Bool, /// - /// Integer header. + /// Signed 8-bit integer header. + /// + Int8, + + /// + /// Signed 16-bit integer header. + /// + Int16, + + /// + /// Signed 32-bit integer header. /// Int32, @@ -53,7 +63,17 @@ public enum HeaderKind Int128, /// - /// Unsigned integer header. + /// Unsigned 8-bit integer header. + /// + Uint8, + + /// + /// Unsigned 16-bit integer header. + /// + Uint16, + + /// + /// Unsigned 32-bit integer header. /// Uint32, diff --git a/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs b/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs index 388809e96b..681849fc48 100644 --- a/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs +++ b/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs @@ -250,9 +250,13 @@ public byte[] ToBytes() HeaderKind.Raw => Value.ToString(), HeaderKind.String => Encoding.UTF8.GetString(Value), HeaderKind.Bool => Value[0].ToString(CultureInfo.InvariantCulture), + HeaderKind.Int8 => ((sbyte)Value[0]).ToString(), + HeaderKind.Int16 => BinaryPrimitives.ReadInt16LittleEndian(Value).ToString(), HeaderKind.Int32 => BinaryPrimitives.ReadInt32LittleEndian(Value).ToString(), HeaderKind.Int64 => BinaryPrimitives.ReadInt64LittleEndian(Value).ToString(), HeaderKind.Int128 => Value.ToInt128().ToString(), + HeaderKind.Uint8 => Value[0].ToString(), + HeaderKind.Uint16 => BinaryPrimitives.ReadUInt16LittleEndian(Value).ToString(), HeaderKind.Uint32 => BinaryPrimitives.ReadUInt32LittleEndian(Value).ToString(), HeaderKind.Uint64 => BinaryPrimitives.ReadUInt64LittleEndian(Value).ToString(), HeaderKind.Uint128 => Value.ToUInt128().ToString(), diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index 699baf2096..4e46b47dec 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs @@ -55,7 +55,10 @@ internal HttpMessageStream(HttpClient httpClient) _jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, - Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } + Converters = + { + new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) + } }; } diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj index ffc1c58f71..cf7adcb5fe 100644 --- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj +++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj @@ -7,7 +7,7 @@ net8.0;net10.0 Apache.Iggy Apache.Iggy - 0.6.1-edge.1 + 0.6.2-edge.1 true diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/HeaderKeyConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/HeaderKeyConverter.cs deleted file mode 100644 index 8221796d5e..0000000000 --- a/foreign/csharp/Iggy_SDK/JsonConverters/HeaderKeyConverter.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text.Json; -using System.Text.Json.Serialization; -using Apache.Iggy.Headers; - -namespace Apache.Iggy.JsonConverters; - -internal class HeaderKeyConverter : JsonConverter -{ - public override HeaderKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - return HeaderKey.New(reader.GetString() ?? throw new JsonException("Header key cannot be null or empty.")); - } - - public override void Write(Utf8JsonWriter writer, HeaderKey value, JsonSerializerOptions options) - { - writer.WriteStringValue(value.Value); - } - - public override HeaderKey ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, - JsonSerializerOptions options) - { - return HeaderKey.New(reader.GetString() ?? throw new JsonException("Header key cannot be null or empty.")); - } - - public override void WriteAsPropertyName(Utf8JsonWriter writer, HeaderKey value, JsonSerializerOptions options) - { - writer.WritePropertyName(value.Value); - } -} diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/MessageConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/MessageConverter.cs index e154f6a8eb..ad8d721f44 100644 --- a/foreign/csharp/Iggy_SDK/JsonConverters/MessageConverter.cs +++ b/foreign/csharp/Iggy_SDK/JsonConverters/MessageConverter.cs @@ -54,38 +54,51 @@ private static void WriteHeaders(Utf8JsonWriter writer, Dictionary "raw", + HeaderKind.String => "string", HeaderKind.Bool => "bool", + HeaderKind.Int8 => "int8", + HeaderKind.Int16 => "int16", HeaderKind.Int32 => "int32", HeaderKind.Int64 => "int64", HeaderKind.Int128 => "int128", + HeaderKind.Uint8 => "uint8", + HeaderKind.Uint16 => "uint16", HeaderKind.Uint32 => "uint32", HeaderKind.Uint64 => "uint64", HeaderKind.Uint128 => "uint128", HeaderKind.Float => "float32", HeaderKind.Double => "float64", - HeaderKind.String => "string", - HeaderKind.Raw => "raw", _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "Invalid header kind") }; } diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs new file mode 100644 index 0000000000..35aa32688a --- /dev/null +++ b/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; + +namespace Apache.Iggy.JsonConverters; + +internal class UserHeadersConverter : JsonConverter?> +{ + public override Dictionary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new JsonException($"Expected start of array for headers but got {reader.TokenType}."); + } + + var result = new Dictionary(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + { + return result.Count == 0 ? null : result; + } + + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected start of object for header entry."); + } + + HeaderKey? key = null; + HeaderValue? value = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException("Expected property name."); + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "key": + key = ReadHeaderKey(ref reader); + break; + case "value": + value = ReadHeaderValue(ref reader); + break; + } + } + + if (key is null || value is null) + { + throw new JsonException("Header entry must have both 'key' and 'value' properties."); + } + + result[key.Value] = value.Value; + } + + return result; + } + + private static HeaderKey ReadHeaderKey(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected start of object for header key."); + } + + HeaderKind? kind = null; + byte[]? value = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException("Expected property name."); + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "kind": + kind = ParseHeaderKind(reader.GetString()); + break; + case "value": + var base64 = reader.GetString(); + value = base64 is not null ? Convert.FromBase64String(base64) : null; + break; + } + } + + if (kind is null || value is null) + { + throw new JsonException("Header key must have both 'kind' and 'value' properties."); + } + + return new HeaderKey { Kind = kind.Value, Value = value }; + } + + private static HeaderValue ReadHeaderValue(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected start of object for header value."); + } + + HeaderKind? kind = null; + byte[]? value = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException("Expected property name."); + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "kind": + kind = ParseHeaderKind(reader.GetString()); + break; + case "value": + var base64 = reader.GetString(); + value = base64 is not null ? Convert.FromBase64String(base64) : null; + break; + } + } + + if (kind is null || value is null) + { + throw new JsonException("Header value must have both 'kind' and 'value' properties."); + } + + return new HeaderValue { Kind = kind.Value, Value = value }; + } + + private static HeaderKind ParseHeaderKind(string? kindStr) + { + return kindStr switch + { + "raw" => HeaderKind.Raw, + "string" => HeaderKind.String, + "bool" => HeaderKind.Bool, + "int8" => HeaderKind.Int8, + "int16" => HeaderKind.Int16, + "int32" => HeaderKind.Int32, + "int64" => HeaderKind.Int64, + "int128" => HeaderKind.Int128, + "uint8" => HeaderKind.Uint8, + "uint16" => HeaderKind.Uint16, + "uint32" => HeaderKind.Uint32, + "uint64" => HeaderKind.Uint64, + "uint128" => HeaderKind.Uint128, + "float32" => HeaderKind.Float, + "float64" => HeaderKind.Double, + _ => throw new JsonException($"Unknown header kind: {kindStr}") + }; + } + + public override void Write(Utf8JsonWriter writer, Dictionary? value, JsonSerializerOptions options) + { + if (value is null) + { + writer.WriteNullValue(); + return; + } + + writer.WriteStartArray(); + + foreach (var kvp in value) + { + writer.WriteStartObject(); + + writer.WriteStartObject("key"); + writer.WriteString("kind", GetHeaderKindString(kvp.Key.Kind)); + writer.WriteBase64String("value", kvp.Key.Value); + writer.WriteEndObject(); + + writer.WriteStartObject("value"); + writer.WriteString("kind", GetHeaderKindString(kvp.Value.Kind)); + writer.WriteBase64String("value", kvp.Value.Value); + writer.WriteEndObject(); + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + private static string GetHeaderKindString(HeaderKind kind) + { + return kind switch + { + HeaderKind.Raw => "raw", + HeaderKind.String => "string", + HeaderKind.Bool => "bool", + HeaderKind.Int8 => "int8", + HeaderKind.Int16 => "int16", + HeaderKind.Int32 => "int32", + HeaderKind.Int64 => "int64", + HeaderKind.Int128 => "int128", + HeaderKind.Uint8 => "uint8", + HeaderKind.Uint16 => "uint16", + HeaderKind.Uint32 => "uint32", + HeaderKind.Uint64 => "uint64", + HeaderKind.Uint128 => "uint128", + HeaderKind.Float => "float32", + HeaderKind.Double => "float64", + _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "Invalid header kind") + }; + } +} diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index 288a0fb5cc..63b8376a93 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -428,17 +428,22 @@ private static Dictionary MapHeaders(ReadOnlySpan while (position < payload.Length) { + var keyKind = MapHeaderKind(payload, position); + position++; + var keyLength = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); if (keyLength is 0 or > 255) { throw new ArgumentException("Key has incorrect size, must be between 1 and 255", nameof(keyLength)); } - var key = Encoding.UTF8.GetString(payload[(position + 4)..(position + 4 + keyLength)]); - position += 4 + keyLength; + position += 4; + var keyValue = payload[position..(position + keyLength)].ToArray(); + position += keyLength; - var headerKind = MapHeaderKind(payload, position); + var valueKind = MapHeaderKind(payload, position); position++; + var valueLength = BinaryPrimitives.ReadInt32LittleEndian(payload[position..(position + 4)]); if (valueLength is 0 or > 255) { @@ -448,11 +453,9 @@ private static Dictionary MapHeaders(ReadOnlySpan position += 4; ReadOnlySpan value = payload[position..(position + valueLength)]; position += valueLength; - headers.Add(HeaderKey.New(key), new HeaderValue - { - Kind = headerKind, - Value = value.ToArray() - }); + + headers[new HeaderKey { Kind = keyKind, Value = keyValue }] = + new HeaderValue { Kind = valueKind, Value = value.ToArray() }; } return headers; @@ -465,9 +468,13 @@ private static HeaderKind MapHeaderKind(ReadOnlySpan payload, int position 1 => HeaderKind.Raw, 2 => HeaderKind.String, 3 => HeaderKind.Bool, + 4 => HeaderKind.Int8, + 5 => HeaderKind.Int16, 6 => HeaderKind.Int32, 7 => HeaderKind.Int64, 8 => HeaderKind.Int128, + 9 => HeaderKind.Uint8, + 10 => HeaderKind.Uint16, 11 => HeaderKind.Uint32, 12 => HeaderKind.Uint64, 13 => HeaderKind.Uint128, diff --git a/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs b/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs index 7f98ae73e9..22445d37d6 100644 --- a/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs +++ b/foreign/csharp/Iggy_SDK/Utils/TcpMessageStreamHelpers.cs @@ -56,7 +56,7 @@ internal static int CalculateMessageBytesCount(IList messages) foreach (var header in message.UserHeaders) { - bytesCount += 4 + header.Key.Value.Length + 1 + 4 + header.Value.Value.Length; + bytesCount += 1 + 4 + header.Key.Value.Length + 1 + 4 + header.Value.Value.Length; } } diff --git a/foreign/go/binary_serialization/send_messages_request_serializer_test.go b/foreign/go/binary_serialization/send_messages_request_serializer_test.go index 29807c3433..985c4b4ebf 100644 --- a/foreign/go/binary_serialization/send_messages_request_serializer_test.go +++ b/foreign/go/binary_serialization/send_messages_request_serializer_test.go @@ -54,7 +54,7 @@ func TestSerialize_SendMessagesRequest(t *testing.T) { 0x04, // Partitioning Length 0x01, 0x00, 0x00, 0x00, // PartitionId (123) 0x01, 0x0, 0x0, 0x0, // MessageCount - 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Index (16*1) bytes + 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Index (16*1) bytes } expected = append(expected, message1.Header.ToBytes()...) expected = append(expected, message1.Payload...) @@ -66,10 +66,10 @@ func TestSerialize_SendMessagesRequest(t *testing.T) { } } -func createDefaultMessageHeaders() map[iggcon.HeaderKey]iggcon.HeaderValue { - return map[iggcon.HeaderKey]iggcon.HeaderValue{ - {Value: "HeaderKey1"}: {Kind: iggcon.String, Value: []byte("Value 1")}, - {Value: "HeaderKey2"}: {Kind: iggcon.Uint32, Value: []byte{0x01, 0x02, 0x03, 0x04}}, +func createDefaultMessageHeaders() []iggcon.HeaderEntry { + return []iggcon.HeaderEntry{ + {Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("HeaderKey1")}, Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("Value 1")}}, + {Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("HeaderKey2")}, Value: iggcon.HeaderValue{Kind: iggcon.Uint32, Value: []byte{0x01, 0x02, 0x03, 0x04}}}, } } diff --git a/foreign/go/contracts/messages.go b/foreign/go/contracts/messages.go index 24d87e81b6..4dc3d14a5a 100644 --- a/foreign/go/contracts/messages.go +++ b/foreign/go/contracts/messages.go @@ -118,7 +118,7 @@ func WithID(id [16]byte) IggyMessageOpt { } } -func WithUserHeaders(userHeaders map[HeaderKey]HeaderValue) IggyMessageOpt { +func WithUserHeaders(userHeaders []HeaderEntry) IggyMessageOpt { return func(m *IggyMessage) { userHeaderBytes := GetHeadersBytes(userHeaders) m.UserHeaders = userHeaderBytes diff --git a/foreign/go/contracts/user_headers.go b/foreign/go/contracts/user_headers.go index 3349c895c2..2d678063f7 100644 --- a/foreign/go/contracts/user_headers.go +++ b/foreign/go/contracts/user_headers.go @@ -28,14 +28,33 @@ type HeaderValue struct { } type HeaderKey struct { - Value string + Kind HeaderKind + Value []byte +} + +type HeaderEntry struct { + Key HeaderKey + Value HeaderValue } -func NewHeaderKey(val string) (HeaderKey, error) { +func NewHeaderKeyString(val string) (HeaderKey, error) { if len(val) == 0 || len(val) > 255 { return HeaderKey{}, errors.New("value has incorrect size, must be between 1 and 255") } - return HeaderKey{Value: val}, nil + return HeaderKey{Kind: String, Value: []byte(val)}, nil +} + +func NewHeaderKeyRaw(val []byte) (HeaderKey, error) { + if len(val) == 0 || len(val) > 255 { + return HeaderKey{}, errors.New("value has incorrect size, must be between 1 and 255") + } + return HeaderKey{Kind: Raw, Value: val}, nil +} + +func NewHeaderKeyInt32(val int32) HeaderKey { + buf := make([]byte, 4) + binary.LittleEndian.PutUint32(buf, uint32(val)) + return HeaderKey{Kind: Int32, Value: buf} } type HeaderKind int @@ -58,15 +77,32 @@ const ( Double HeaderKind = 15 ) -func GetHeadersBytes(headers map[HeaderKey]HeaderValue) []byte { +func (k HeaderKind) ExpectedSize() int { + switch k { + case Bool, Int8, Uint8: + return 1 + case Int16, Uint16: + return 2 + case Int32, Uint32, Float: + return 4 + case Int64, Uint64, Double: + return 8 + case Int128, Uint128: + return 16 + default: + return -1 + } +} + +func GetHeadersBytes(headers []HeaderEntry) []byte { headersLength := 0 - for key, header := range headers { - headersLength += 4 + len(key.Value) + 1 + 4 + len(header.Value) + for _, entry := range headers { + headersLength += 1 + 4 + len(entry.Key.Value) + 1 + 4 + len(entry.Value.Value) } headersBytes := make([]byte, headersLength) position := 0 - for key, value := range headers { - headerBytes := getBytesFromHeader(key, value) + for _, entry := range headers { + headerBytes := getBytesFromHeader(entry.Key, entry.Value) copy(headersBytes[position:position+len(headerBytes)], headerBytes) position += len(headerBytes) } @@ -74,25 +110,39 @@ func GetHeadersBytes(headers map[HeaderKey]HeaderValue) []byte { } func getBytesFromHeader(key HeaderKey, value HeaderValue) []byte { - headerBytesLength := 4 + len(key.Value) + 1 + 4 + len(value.Value) + headerBytesLength := 1 + 4 + len(key.Value) + 1 + 4 + len(value.Value) headerBytes := make([]byte, headerBytesLength) + pos := 0 + + headerBytes[pos] = byte(key.Kind) + pos++ - binary.LittleEndian.PutUint32(headerBytes[:4], uint32(len(key.Value))) - copy(headerBytes[4:4+len(key.Value)], key.Value) + binary.LittleEndian.PutUint32(headerBytes[pos:pos+4], uint32(len(key.Value))) + pos += 4 + copy(headerBytes[pos:pos+len(key.Value)], key.Value) + pos += len(key.Value) - headerBytes[4+len(key.Value)] = byte(value.Kind) + headerBytes[pos] = byte(value.Kind) + pos++ - binary.LittleEndian.PutUint32(headerBytes[4+len(key.Value)+1:4+len(key.Value)+1+4], uint32(len(value.Value))) - copy(headerBytes[4+len(key.Value)+1+4:], value.Value) + binary.LittleEndian.PutUint32(headerBytes[pos:pos+4], uint32(len(value.Value))) + pos += 4 + copy(headerBytes[pos:], value.Value) return headerBytes } -func DeserializeHeaders(userHeadersBytes []byte) (map[HeaderKey]HeaderValue, error) { - headers := make(map[HeaderKey]HeaderValue) +func DeserializeHeaders(userHeadersBytes []byte) ([]HeaderEntry, error) { + var headers []HeaderEntry position := 0 for position < len(userHeadersBytes) { + keyKind, err := deserializeHeaderKind(userHeadersBytes, position) + if err != nil { + return nil, err + } + position++ + if len(userHeadersBytes) <= position+4 { return nil, errors.New("invalid header key length") } @@ -107,10 +157,15 @@ func DeserializeHeaders(userHeadersBytes []byte) (map[HeaderKey]HeaderValue, err return nil, errors.New("invalid header key") } - key := string(userHeadersBytes[position : position+int(keyLength)]) + if expected := keyKind.ExpectedSize(); expected != -1 && int(keyLength) != expected { + return nil, errors.New("invalid header key size for kind") + } + + keyValue := make([]byte, keyLength) + copy(keyValue, userHeadersBytes[position:position+int(keyLength)]) position += int(keyLength) - headerKind, err := deserializeHeaderKind(userHeadersBytes, position) + valueKind, err := deserializeHeaderKind(userHeadersBytes, position) if err != nil { return nil, err } @@ -131,13 +186,18 @@ func DeserializeHeaders(userHeadersBytes []byte) (map[HeaderKey]HeaderValue, err return nil, errors.New("invalid header value") } - value := userHeadersBytes[position : position+int(valueLength)] + if expected := valueKind.ExpectedSize(); expected != -1 && int(valueLength) != expected { + return nil, errors.New("invalid header value size for kind") + } + + valueBytes := make([]byte, valueLength) + copy(valueBytes, userHeadersBytes[position:position+int(valueLength)]) position += int(valueLength) - headers[HeaderKey{Value: key}] = HeaderValue{ - Kind: headerKind, - Value: value, - } + headers = append(headers, HeaderEntry{ + Key: HeaderKey{Kind: keyKind, Value: keyValue}, + Value: HeaderValue{Kind: valueKind, Value: valueBytes}, + }) } return headers, nil diff --git a/foreign/java/gradle.properties b/foreign/java/gradle.properties index 5ff6760601..2942656f95 100644 --- a/foreign/java/gradle.properties +++ b/foreign/java/gradle.properties @@ -17,5 +17,5 @@ # under the License. # -version=0.6.1-SNAPSHOT +version=0.6.2-SNAPSHOT group=org.apache.iggy diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ObjectMapperFactory.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ObjectMapperFactory.java index 609a3a919d..cd7a8e9b9f 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ObjectMapperFactory.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ObjectMapperFactory.java @@ -29,7 +29,7 @@ import java.util.Map; -final class ObjectMapperFactory { +public final class ObjectMapperFactory { private static final ObjectMapper INSTANCE = JsonMapper.builder() .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) @@ -40,7 +40,7 @@ final class ObjectMapperFactory { private ObjectMapperFactory() {} - static ObjectMapper getInstance() { + public static ObjectMapper getInstance() { return INSTANCE; } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderEntry.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderEntry.java new file mode 100644 index 0000000000..cec2175461 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderEntry.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.message; + +/** + * Represents a single header entry with key and value. + * Used for JSON serialization/deserialization of user headers. + */ +public record HeaderEntry(HeaderKey key, HeaderValue value) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKey.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKey.java new file mode 100644 index 0000000000..ad65857a93 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKey.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.message; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; + +public record HeaderKey(HeaderKind kind, byte[] value) { + @JsonCreator + public static HeaderKey fromJson(@JsonProperty("kind") HeaderKind kind, @JsonProperty("value") String base64Value) { + byte[] decodedValue = Base64.getDecoder().decode(base64Value); + return new HeaderKey(kind, decodedValue); + } + + public static HeaderKey fromString(String val) { + if (val.isEmpty() || val.length() > 255) { + throw new IllegalArgumentException("Value has incorrect size, must be between 1 and 255"); + } + return new HeaderKey(HeaderKind.String, val.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HeaderKey headerKey = (HeaderKey) o; + return kind == headerKey.kind && Arrays.equals(value, headerKey.value); + } + + @Override + public int hashCode() { + int result = kind.hashCode(); + result = 31 * result + Arrays.hashCode(value); + return result; + } + + @Override + public String toString() { + return new String(value, StandardCharsets.UTF_8); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKind.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKind.java index 00f024c25b..753a9ee5a9 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKind.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderKind.java @@ -19,23 +19,39 @@ package org.apache.iggy.message; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.iggy.exception.IggyInvalidArgumentException; public enum HeaderKind { + @JsonProperty("raw") Raw(1), + @JsonProperty("string") String(2), + @JsonProperty("bool") Bool(3), + @JsonProperty("int8") Int8(4), + @JsonProperty("int16") Int16(5), + @JsonProperty("int32") Int32(6), + @JsonProperty("int64") Int64(7), + @JsonProperty("int128") Int128(8), + @JsonProperty("uint8") Uint8(9), + @JsonProperty("uint16") Uint16(10), + @JsonProperty("uint32") Uint32(11), + @JsonProperty("uint64") Uint64(12), + @JsonProperty("uint128") Uint128(13), + @JsonProperty("float32") Float32(14), + @JsonProperty("float64") Float64(15); private final int code; diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java index 3714dd27e4..f4fc6a4c28 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java @@ -19,4 +19,227 @@ package org.apache.iggy.message; -public record HeaderValue(HeaderKind kind, String value) {} +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; + +public record HeaderValue(HeaderKind kind, byte[] value) { + @JsonCreator + public static HeaderValue fromJson( + @JsonProperty("kind") HeaderKind kind, @JsonProperty("value") String base64Value) { + byte[] decodedValue = Base64.getDecoder().decode(base64Value); + return new HeaderValue(kind, decodedValue); + } + + public static HeaderValue fromString(String val) { + if (val.isEmpty() || val.length() > 255) { + throw new IllegalArgumentException("Value has incorrect size, must be between 1 and 255"); + } + return new HeaderValue(HeaderKind.String, val.getBytes(StandardCharsets.UTF_8)); + } + + public static HeaderValue fromBool(boolean val) { + return new HeaderValue(HeaderKind.Bool, new byte[] {(byte) (val ? 1 : 0)}); + } + + public static HeaderValue fromInt8(byte val) { + return new HeaderValue(HeaderKind.Int8, new byte[] {val}); + } + + public static HeaderValue fromInt16(short val) { + ByteBuffer buffer = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN); + buffer.putShort(val); + return new HeaderValue(HeaderKind.Int16, buffer.array()); + } + + public static HeaderValue fromInt32(int val) { + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(val); + return new HeaderValue(HeaderKind.Int32, buffer.array()); + } + + public static HeaderValue fromInt64(long val) { + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong(val); + return new HeaderValue(HeaderKind.Int64, buffer.array()); + } + + public static HeaderValue fromUint8(short val) { + if (val < 0 || val > 255) { + throw new IllegalArgumentException("Value must be between 0 and 255"); + } + return new HeaderValue(HeaderKind.Uint8, new byte[] {(byte) val}); + } + + public static HeaderValue fromUint16(int val) { + if (val < 0 || val > 65535) { + throw new IllegalArgumentException("Value must be between 0 and 65535"); + } + ByteBuffer buffer = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN); + buffer.putShort((short) val); + return new HeaderValue(HeaderKind.Uint16, buffer.array()); + } + + public static HeaderValue fromUint32(long val) { + if (val < 0 || val > 4294967295L) { + throw new IllegalArgumentException("Value must be between 0 and 4294967295"); + } + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt((int) val); + return new HeaderValue(HeaderKind.Uint32, buffer.array()); + } + + public static HeaderValue fromFloat32(float val) { + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putFloat(val); + return new HeaderValue(HeaderKind.Float32, buffer.array()); + } + + public static HeaderValue fromFloat64(double val) { + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(val); + return new HeaderValue(HeaderKind.Float64, buffer.array()); + } + + public static HeaderValue fromRaw(byte[] val) { + if (val.length == 0 || val.length > 255) { + throw new IllegalArgumentException("Value has incorrect size, must be between 1 and 255"); + } + return new HeaderValue(HeaderKind.Raw, val); + } + + public String asString() { + if (kind != HeaderKind.String) { + throw new IllegalStateException("Header value is not a string, kind: " + kind); + } + return new String(value, StandardCharsets.UTF_8); + } + + public boolean asBool() { + if (kind != HeaderKind.Bool) { + throw new IllegalStateException("Header value is not a bool, kind: " + kind); + } + return value[0] == 1; + } + + public byte asInt8() { + if (kind != HeaderKind.Int8) { + throw new IllegalStateException("Header value is not an int8, kind: " + kind); + } + return value[0]; + } + + public short asInt16() { + if (kind != HeaderKind.Int16) { + throw new IllegalStateException("Header value is not an int16, kind: " + kind); + } + return ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getShort(); + } + + public int asInt32() { + if (kind != HeaderKind.Int32) { + throw new IllegalStateException("Header value is not an int32, kind: " + kind); + } + return ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getInt(); + } + + public long asInt64() { + if (kind != HeaderKind.Int64) { + throw new IllegalStateException("Header value is not an int64, kind: " + kind); + } + return ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getLong(); + } + + public short asUint8() { + if (kind != HeaderKind.Uint8) { + throw new IllegalStateException("Header value is not a uint8, kind: " + kind); + } + return (short) (value[0] & 0xFF); + } + + public int asUint16() { + if (kind != HeaderKind.Uint16) { + throw new IllegalStateException("Header value is not a uint16, kind: " + kind); + } + return (ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getShort() & 0xFFFF); + } + + public long asUint32() { + if (kind != HeaderKind.Uint32) { + throw new IllegalStateException("Header value is not a uint32, kind: " + kind); + } + return (ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getInt() & 0xFFFFFFFFL); + } + + public float asFloat32() { + if (kind != HeaderKind.Float32) { + throw new IllegalStateException("Header value is not a float32, kind: " + kind); + } + return ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getFloat(); + } + + public double asFloat64() { + if (kind != HeaderKind.Float64) { + throw new IllegalStateException("Header value is not a float64, kind: " + kind); + } + return ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getDouble(); + } + + public byte[] asRaw() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HeaderValue that = (HeaderValue) o; + return kind == that.kind && Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + int result = kind.hashCode(); + result = 31 * result + Arrays.hashCode(value); + return result; + } + + @Override + public String toString() { + return toStringValue(); + } + + private String toStringValue() { + if (kind == HeaderKind.String) { + return asString(); + } + if (kind == HeaderKind.Bool) { + return String.valueOf(asBool()); + } + return numericOrRawToString(); + } + + private String numericOrRawToString() { + return switch (kind) { + case Int8 -> String.valueOf(asInt8()); + case Int16 -> String.valueOf(asInt16()); + case Int32 -> String.valueOf(asInt32()); + case Int64 -> String.valueOf(asInt64()); + case Uint8 -> String.valueOf(asUint8()); + case Uint16 -> String.valueOf(asUint16()); + case Uint32 -> String.valueOf(asUint32()); + case Float32 -> String.valueOf(asFloat32()); + case Float64 -> String.valueOf(asFloat64()); + default -> Base64.getEncoder().encodeToString(value); + }; + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Message.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Message.java index 4e78293a1f..144103d102 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Message.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Message.java @@ -19,18 +19,46 @@ package org.apache.iggy.message; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; +import org.apache.iggy.serde.Base64Serializer; +import org.apache.iggy.serde.UserHeadersSerializer; +import tools.jackson.databind.annotation.JsonSerialize; + import java.math.BigInteger; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; -public record Message(MessageHeader header, byte[] payload, Map userHeaders) { +public record Message( + MessageHeader header, + @JsonSerialize(using = Base64Serializer.class) byte[] payload, + @JsonSerialize(using = UserHeadersSerializer.class) Map userHeaders) { + @JsonCreator + public static Message fromJson( + @JsonProperty("header") MessageHeader header, + @JsonProperty("payload") String base64Payload, + @JsonProperty(value = "user_headers", required = false) @JsonSetter(nulls = Nulls.AS_EMPTY) + List userHeadersList) { + byte[] decodedPayload = Base64.getDecoder().decode(base64Payload); + Map headersMap = new HashMap<>(); + if (userHeadersList != null) { + for (HeaderEntry entry : userHeadersList) { + headersMap.put(entry.key(), entry.value()); + } + } + return new Message(header, decodedPayload, headersMap); + } public static Message of(String payload) { return of(payload, Collections.emptyMap()); } - public static Message of(String payload, Map userHeaders) { + public static Message of(String payload, Map userHeaders) { final byte[] payloadBytes = payload.getBytes(); final long userHeadersLength = getUserHeadersSize(userHeaders); final MessageHeader msgHeader = new MessageHeader( @@ -44,8 +72,8 @@ public static Message of(String payload, Map userHeaders) { return new Message(msgHeader, payloadBytes, userHeaders); } - public Message withUserHeaders(Map userHeaders) { - Map mergedHeaders = mergeUserHeaders(userHeaders); + public Message withUserHeaders(Map userHeaders) { + Map mergedHeaders = mergeUserHeaders(userHeaders); long userHeadersLength = getUserHeadersSize(mergedHeaders); MessageHeader updatedHeader = new MessageHeader( header.checksum(), @@ -63,7 +91,7 @@ public int getSize() { return Math.toIntExact(MessageHeader.SIZE + payload.length + userHeadersLength); } - private Map mergeUserHeaders(Map userHeaders) { + private Map mergeUserHeaders(Map userHeaders) { if (userHeaders.isEmpty()) { return this.userHeaders; } @@ -72,21 +100,21 @@ private Map mergeUserHeaders(Map userH return userHeaders; } - Map mergedHeaders = new HashMap<>(this.userHeaders); + Map mergedHeaders = new HashMap<>(this.userHeaders); mergedHeaders.putAll(userHeaders); return mergedHeaders; } - private static long getUserHeadersSize(Map userHeaders) { + private static long getUserHeadersSize(Map userHeaders) { if (userHeaders.isEmpty()) { return 0L; } long size = 0L; - for (Map.Entry entry : userHeaders.entrySet()) { - byte[] keyBytes = entry.getKey().getBytes(); - byte[] valueBytes = entry.getValue().value().getBytes(); - size += 4L + keyBytes.length + 1L + 4L + valueBytes.length; + for (Map.Entry entry : userHeaders.entrySet()) { + byte[] keyBytes = entry.getKey().value(); + byte[] valueBytes = entry.getValue().value(); + size += 1L + 4L + keyBytes.length + 1L + 4L + valueBytes.length; } return size; } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Partitioning.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Partitioning.java index c725eb4eea..c451510d22 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Partitioning.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/Partitioning.java @@ -21,11 +21,14 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.iggy.exception.IggyInvalidArgumentException; +import org.apache.iggy.serde.Base64Serializer; +import tools.jackson.databind.annotation.JsonSerialize; import java.nio.ByteBuffer; -public record Partitioning(PartitioningKind kind, byte[] value) { - +public record Partitioning( + PartitioningKind kind, + @JsonSerialize(using = Base64Serializer.class) byte[] value) { public static Partitioning balanced() { return new Partitioning(PartitioningKind.Balanced, new byte[] {}); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/Base64Serializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/Base64Serializer.java new file mode 100644 index 0000000000..17bda8b396 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/Base64Serializer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.serde; + +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ValueSerializer; + +import java.util.Base64; + +public class Base64Serializer extends ValueSerializer { + + @Override + public void serialize(byte[] value, JsonGenerator gen, SerializationContext ctxt) throws JacksonException { + gen.writeString(Base64.getEncoder().encodeToString(value)); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java index 378b8930cc..fcd16ca55e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java @@ -26,6 +26,7 @@ import org.apache.iggy.consumergroup.ConsumerGroupMember; import org.apache.iggy.consumeroffset.ConsumerOffsetInfo; import org.apache.iggy.message.BytesMessageId; +import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderKind; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.Message; @@ -196,21 +197,24 @@ public static Message readPolledMessage(ByteBuf response) { new MessageHeader(checksum, id, offset, timestamp, originTimestamp, userHeadersLength, payloadLength); var payload = newByteArray(payloadLength); response.readBytes(payload); - Map userHeaders = new HashMap<>(); + Map userHeaders = new HashMap<>(); if (userHeadersLength > 0) { ByteBuf userHeadersBuffer = response.readSlice(toInt(userHeadersLength)); - Map headers = new HashMap<>(); + Map headers = new HashMap<>(); while (userHeadersBuffer.isReadable()) { + var userHeaderKeyKindCode = userHeadersBuffer.readUnsignedByte(); var userHeaderKeyLength = userHeadersBuffer.readUnsignedIntLE(); - var userHeaderKey = userHeadersBuffer - .readCharSequence(toInt(userHeaderKeyLength), StandardCharsets.UTF_8) - .toString(); - var userHeaderKindCode = userHeadersBuffer.readUnsignedByte(); + byte[] userHeaderKeyBytes = new byte[toInt(userHeaderKeyLength)]; + userHeadersBuffer.readBytes(userHeaderKeyBytes); + var userHeaderKey = new HeaderKey(HeaderKind.fromCode(userHeaderKeyKindCode), userHeaderKeyBytes); + + var userHeaderValueKindCode = userHeadersBuffer.readUnsignedByte(); var userHeaderValueLength = userHeadersBuffer.readUnsignedIntLE(); - String userHeaderValue = userHeadersBuffer - .readCharSequence(toInt(userHeaderValueLength), StandardCharsets.UTF_8) - .toString(); - headers.put(userHeaderKey, new HeaderValue(HeaderKind.fromCode(userHeaderKindCode), userHeaderValue)); + byte[] userHeaderValueBytes = new byte[toInt(userHeaderValueLength)]; + userHeadersBuffer.readBytes(userHeaderValueBytes); + headers.put( + userHeaderKey, + new HeaderValue(HeaderKind.fromCode(userHeaderValueKindCode), userHeaderValueBytes)); } userHeaders = headers; } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java index 9971575ee6..20301ecdbd 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesSerializer.java @@ -25,6 +25,7 @@ import org.apache.iggy.consumergroup.Consumer; import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.identifier.Identifier; +import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.Message; import org.apache.iggy.message.MessageHeader; @@ -120,20 +121,21 @@ public static ByteBuf toBytes(Optional optionalLong) { return buffer; } - public static ByteBuf toBytes(Map headers) { + public static ByteBuf toBytes(Map headers) { if (headers.isEmpty()) { return Unpooled.EMPTY_BUFFER; } var buffer = Unpooled.buffer(); - for (Map.Entry entry : headers.entrySet()) { - String key = entry.getKey(); - buffer.writeIntLE(key.length()); - buffer.writeBytes(key.getBytes()); + for (Map.Entry entry : headers.entrySet()) { + HeaderKey key = entry.getKey(); + buffer.writeByte(key.kind().asCode()); + buffer.writeIntLE(key.value().length); + buffer.writeBytes(key.value()); HeaderValue value = entry.getValue(); buffer.writeByte(value.kind().asCode()); - buffer.writeIntLE(value.value().length()); - buffer.writeBytes(value.value().getBytes()); + buffer.writeIntLE(value.value().length); + buffer.writeBytes(value.value()); } return buffer; } @@ -217,7 +219,7 @@ public static ByteBuf toBytesAsU64(BigInteger value) { } ByteBuf buffer = Unpooled.buffer(8, 8); byte[] valueAsBytes = value.toByteArray(); - if (valueAsBytes.length > 9 || valueAsBytes.length == 9 && valueAsBytes[0] != 0) { + if (valueAsBytes.length > 9 || (valueAsBytes.length == 9 && valueAsBytes[0] != 0)) { throw new IggyInvalidArgumentException("Value too large for U64: " + value); } ArrayUtils.reverse(valueAsBytes); @@ -234,7 +236,7 @@ public static ByteBuf toBytesAsU128(BigInteger value) { } ByteBuf buffer = Unpooled.buffer(16, 16); byte[] valueAsBytes = value.toByteArray(); - if (valueAsBytes.length > 17 || valueAsBytes.length == 17 && valueAsBytes[0] != 0) { + if (valueAsBytes.length > 17 || (valueAsBytes.length == 17 && valueAsBytes[0] != 0)) { throw new IggyInvalidArgumentException("Value too large for U128: " + value); } ArrayUtils.reverse(valueAsBytes); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/UserHeadersSerializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/UserHeadersSerializer.java new file mode 100644 index 0000000000..878ad86701 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/UserHeadersSerializer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.serde; + +import org.apache.iggy.message.HeaderEntry; +import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderValue; +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonGenerator; +import tools.jackson.databind.SerializationContext; +import tools.jackson.databind.ValueSerializer; + +import java.util.Map; + +public class UserHeadersSerializer extends ValueSerializer> { + + @Override + public void serialize(Map headers, JsonGenerator gen, SerializationContext ctxt) + throws JacksonException { + gen.writeStartArray(); + for (Map.Entry entry : headers.entrySet()) { + ctxt.findValueSerializer(HeaderEntry.class) + .serialize(new HeaderEntry(entry.getKey(), entry.getValue()), gen, ctxt); + } + gen.writeEndArray(); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/BytesSerializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/BytesSerializerTest.java index 09f100cc07..3453f18631 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/BytesSerializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/BytesSerializerTest.java @@ -26,7 +26,7 @@ import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.message.BytesMessageId; -import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.Message; import org.apache.iggy.message.MessageHeader; @@ -445,7 +445,7 @@ class HeadersSerialization { @Test void shouldSerializeEmptyHeaders() { // given - Map headers = new HashMap<>(); + Map headers = new HashMap<>(); // when ByteBuf result = BytesSerializer.toBytes(headers); @@ -457,13 +457,14 @@ void shouldSerializeEmptyHeaders() { @Test void shouldSerializeSingleHeader() { // given - Map headers = new HashMap<>(); - headers.put("key1", new HeaderValue(HeaderKind.Raw, "value1")); + Map headers = new HashMap<>(); + headers.put(HeaderKey.fromString("key1"), HeaderValue.fromRaw("value1".getBytes())); // when ByteBuf result = BytesSerializer.toBytes(headers); // then + assertThat(result.readByte()).isEqualTo((byte) 2); // String kind assertThat(result.readIntLE()).isEqualTo(4); // "key1".length() byte[] keyBytes = new byte[4]; result.readBytes(keyBytes); @@ -478,15 +479,15 @@ void shouldSerializeSingleHeader() { @Test void shouldSerializeMultipleHeaders() { // given - Map headers = new HashMap<>(); - headers.put("k1", new HeaderValue(HeaderKind.Raw, "v1")); // 13 bytes - headers.put("k2", new HeaderValue(HeaderKind.String, "v2")); // 13 bytes + Map headers = new HashMap<>(); + headers.put(HeaderKey.fromString("k1"), HeaderValue.fromRaw("v1".getBytes())); + headers.put(HeaderKey.fromString("k2"), HeaderValue.fromString("v2")); // when ByteBuf result = BytesSerializer.toBytes(headers); // then - verify buffer contains data for both headers - assertThat(result.readableBytes()).isEqualTo(26); + assertThat(result.readableBytes()).isEqualTo(28); } } @@ -520,8 +521,8 @@ void shouldSerializeMessageWithoutUserHeaders() { void shouldSerializeMessageWithUserHeaders() { // given var messageId = new BytesMessageId(new byte[16]); - Map userHeaders = new HashMap<>(); - userHeaders.put("key", new HeaderValue(HeaderKind.Raw, "val")); + Map userHeaders = new HashMap<>(); + userHeaders.put(HeaderKey.fromString("key"), HeaderValue.fromRaw("val".getBytes())); // Calculate user headers size ByteBuf headersBuf = BytesSerializer.toBytes(userHeaders); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java index 4558d30fa6..36a6d5ea36 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java @@ -22,6 +22,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import org.apache.commons.lang3.ArrayUtils; +import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderKind; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.user.UserStatus; @@ -364,10 +365,11 @@ void shouldDeserializePolledMessageWithUserHeaders() { // Calculate and write user headers ByteBuf headersBuffer = Unpooled.buffer(); - headersBuffer.writeIntLE(3); // key length + headersBuffer.writeByte(HeaderKind.String.asCode()); + headersBuffer.writeIntLE(3); headersBuffer.writeBytes("key".getBytes()); headersBuffer.writeByte(HeaderKind.Raw.asCode()); - headersBuffer.writeIntLE(3); // value length + headersBuffer.writeIntLE(3); headersBuffer.writeBytes("val".getBytes()); buffer.writeIntLE(headersBuffer.readableBytes()); // user headers length @@ -380,7 +382,8 @@ void shouldDeserializePolledMessageWithUserHeaders() { // then assertThat(message.userHeaders()).hasSize(1); - assertThat(message.userHeaders().get("key").value()).isEqualTo("val"); + assertThat(message.userHeaders().get(HeaderKey.fromString("key")).asRaw()) + .isEqualTo("val".getBytes()); } @Test @@ -749,4 +752,80 @@ void shouldDeserializePersonalAccessTokenInfoWithoutExpiry() { assertThat(tokenInfo.expiryAt()).isEmpty(); } } + + @Nested + class JsonDeserialization { + + private static final tools.jackson.databind.ObjectMapper MAPPER = + org.apache.iggy.client.blocking.http.ObjectMapperFactory.getInstance(); + + @Test + void shouldDeserializePolledMessagesWithEmptyUserHeaders() throws Exception { + String json = """ + { + "partition_id": 1, + "current_offset": 10, + "count": 1, + "messages": [ + { + "header": { + "checksum": 0, + "id": 42, + "offset": 0, + "timestamp": 0, + "origin_timestamp": 1000, + "user_headers_length": 0, + "payload_length": 4 + }, + "payload": "dGVzdA==", + "user_headers": [] + } + ] + } + """; + + var polledMessages = MAPPER.readValue(json, org.apache.iggy.message.PolledMessages.class); + + assertThat(polledMessages).isNotNull(); + assertThat(polledMessages.messages()).hasSize(1); + assertThat(polledMessages.messages().get(0).userHeaders()).isEmpty(); + } + + @Test + void shouldDeserializePolledMessagesWithUserHeaders() throws Exception { + String json = """ + { + "partition_id": 1, + "current_offset": 10, + "count": 1, + "messages": [ + { + "header": { + "checksum": 0, + "id": 42, + "offset": 0, + "timestamp": 0, + "origin_timestamp": 1000, + "user_headers_length": 62, + "payload_length": 4 + }, + "payload": "dGVzdA==", + "user_headers": [ + { + "key": {"kind": "string", "value": "Y29udGVudC10eXBl"}, + "value": {"kind": "string", "value": "dGV4dC9wbGFpbg=="} + } + ] + } + ] + } + """; + + var polledMessages = MAPPER.readValue(json, org.apache.iggy.message.PolledMessages.class); + + assertThat(polledMessages).isNotNull(); + assertThat(polledMessages.messages()).hasSize(1); + assertThat(polledMessages.messages().get(0).userHeaders()).hasSize(1); + } + } } diff --git a/foreign/node/package.json b/foreign/node/package.json index 8d2eb542b6..e04752b5ae 100644 --- a/foreign/node/package.json +++ b/foreign/node/package.json @@ -1,7 +1,7 @@ { "name": "apache-iggy", "type": "module", - "version": "0.6.1-edge.1", + "version": "0.6.2-edge.1", "description": "Official Apache Iggy NodeJS SDK", "keywords": [ "iggy", diff --git a/foreign/node/src/bdd/message.ts b/foreign/node/src/bdd/message.ts index fceb16b97a..4ee3efeab9 100644 --- a/foreign/node/src/bdd/message.ts +++ b/foreign/node/src/bdd/message.ts @@ -17,27 +17,26 @@ * under the License. */ - -import assert from 'node:assert/strict'; +import assert from "node:assert/strict"; import { When, Then } from "@cucumber/cucumber"; -import { Consumer, PollingStrategy, Partitioning } from '../wire/index.js'; -import type { TestWorld } from './world.js'; +import { Consumer, PollingStrategy, Partitioning } from "../wire/index.js"; +import type { TestWorld } from "./world.js"; const generateTestMessages = (count = 1) => { return [...Array(count)].map((_, i) => ({ id: i + 1, - payload: Buffer.from(`Test message ${i}`) + payload: Buffer.from(`Test message ${i}`), })); -} +}; When( - 'I send {int} messages to stream {int}, topic {int}, partition {int}', + "I send {int} messages to stream {int}, topic {int}, partition {int}", async function ( this: TestWorld, msgCount: number, streamId: number, topicId: number, - partitionId: number + partitionId: number, ) { this.sendMessages = generateTestMessages(msgCount); assert.ok( @@ -45,26 +44,22 @@ When( streamId, topicId, messages: this.sendMessages, - partition: Partitioning.PartitionId(partitionId) - }) + partition: Partitioning.PartitionId(partitionId), + }), ); - } + }, ); - -Then( - 'all messages should be sent successfully', - () => true -); +Then("all messages should be sent successfully", () => true); When( - 'I poll messages from stream {int}, topic {int}, partition {int} starting from offset {int}', + "I poll messages from stream {int}, topic {int}, partition {int} starting from offset {int}", async function ( this: TestWorld, streamId: number, topicId: number, partitionId: number, - offset: number + offset: number, ) { const pollReq = { streamId, @@ -73,46 +68,49 @@ When( partitionId, pollingStrategy: PollingStrategy.Offset(BigInt(offset)), count: 100, - autocommit: true + autocommit: true, }; const { messages } = await this.client.message.poll(pollReq); this.polledMessages = messages; assert.equal(this.polledMessages.length, this.sendMessages.length); - } + }, ); Then( - 'I should receive {int} messages', + "I should receive {int} messages", function (this: TestWorld, msgCount: number) { assert.equal(this.polledMessages.length, msgCount); - } + }, ); Then( - 'the messages should have sequential offsets from {int} to {int}', + "the messages should have sequential offsets from {int} to {int}", function (this: TestWorld, from: number, to: number) { for (let i = from; i < to; i++) { - assert.equal(BigInt(i), this.polledMessages[i].headers.offset) + assert.equal(BigInt(i), this.polledMessages[i].headers.offset); } - } + }, ); Then( - 'each message should have the expected payload content', + "each message should have the expected payload content", function (this: TestWorld) { this.sendMessages.forEach((msg, i) => { - assert.deepEqual(msg.payload.toString(), this.polledMessages[i].payload.toString()); - assert.equal(BigInt(msg.id || 0), this.polledMessages[i].headers.id) - }) - } + assert.deepEqual( + msg.payload.toString(), + this.polledMessages[i].payload.toString(), + ); + assert.equal(BigInt(msg.id || 0), this.polledMessages[i].headers.id); + }); + }, ); Then( - 'the last polled message should match the last sent message', + "the last polled message should match the last sent message", function (this: TestWorld) { const lastSent = this.sendMessages[this.sendMessages.length - 1]; const lastPoll = this.polledMessages[this.polledMessages.length - 1]; assert.deepEqual(lastSent.payload.toString(), lastPoll.payload.toString()); - assert.deepEqual(lastSent.headers || {}, lastPoll.userHeaders); - } + assert.deepEqual(lastSent.headers || [], lastPoll.userHeaders); + }, ); diff --git a/foreign/node/src/examples/stream-file-to-topic.ts b/foreign/node/src/examples/stream-file-to-topic.ts index 676c44d25d..b302b92dd8 100644 --- a/foreign/node/src/examples/stream-file-to-topic.ts +++ b/foreign/node/src/examples/stream-file-to-topic.ts @@ -17,71 +17,94 @@ * under the License. */ - -import { open } from 'node:fs/promises'; -import { resolve } from 'node:path'; -import { pipeline } from 'node:stream/promises'; -import { Writable, type TransformCallback } from 'node:stream'; -import { HeaderValue } from '../index.js'; -import { getClient } from './utils.js'; +import { open } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { Writable, type TransformCallback } from "node:stream"; +import { HeaderValue, HeaderKeyFactory } from "../index.js"; +import { getClient } from "./utils.js"; export const fileToTopic = async ( filepath: string, streamName: string, topicName: string, - highWaterMark = 512 * 1024 + highWaterMark = 512 * 1024, ) => { const cli = getClient(); await cli.stream.ensure(streamName); await cli.topic.ensure(streamName, topicName); const fd = await open(filepath); - const fname = filepath.split('/').pop() || filepath; + const fname = filepath.split("/").pop() || filepath; const st = await fd.stat(); - console.log('FILE/STAT', fname, '~', (st.size / (1024 * 1024)).toFixed(2), 'mb', st); + console.log( + "FILE/STAT", + fname, + "~", + (st.size / (1024 * 1024)).toFixed(2), + "mb", + st, + ); const dStart = Date.now(); try { let idx = 0; await pipeline( - fd.createReadStream({highWaterMark}), + fd.createReadStream({ highWaterMark }), new Writable({ - async write(chunks: Buffer, encoding: BufferEncoding, cb: TransformCallback) { - const messages = [{ - headers: { - fileindex: HeaderValue.Uint32(idx++), - filename: HeaderValue.String(fname) + async write( + chunks: Buffer, + encoding: BufferEncoding, + cb: TransformCallback, + ) { + const messages = [ + { + headers: [ + { + key: HeaderKeyFactory.String("fileindex"), + value: HeaderValue.Uint32(idx++), + }, + { + key: HeaderKeyFactory.String("filename"), + value: HeaderValue.String(fname), + }, + ], + payload: chunks, }, - payload: chunks - }]; + ]; try { - await cli.message.send({ streamId: streamName, topicId: topicName, messages }); + await cli.message.send({ + streamId: streamName, + topicId: topicName, + messages, + }); cb(); } catch (err) { - console.log('WRITE ERR', err, chunks); + console.log("WRITE ERR", err, chunks); cb(err as Error); } - } - }) + }, + }), ); - console.log(`Finished ! took ${Date.now() - dStart}ms`,); + console.log(`Finished ! took ${Date.now() - dStart}ms`); } catch (err) { - console.error('Pipeline failed !', err); + console.error("Pipeline failed !", err); throw err; } }; - const argz = process.argv.slice(2); const [rPath, streamIdStr, topicIdStr] = argz; -if (argz.length < 3 || ['-h', '--help', '?'].includes(argz[0])) { - console.log(`Usage: node stream-file-to-topic.js filePath streamName topicName`) - console.log('got', argz); - console.log('note: this script only accept numerical stream/topic id'); +if (argz.length < 3 || ["-h", "--help", "?"].includes(argz[0])) { + console.log( + `Usage: node stream-file-to-topic.js filePath streamName topicName`, + ); + console.log("got", argz); + console.log("note: this script only accept numerical stream/topic id"); process.exit(1); } @@ -89,12 +112,12 @@ const filepath = resolve(rPath); const streamName = streamIdStr; const topicName = topicIdStr; -console.log('running with params:', { filepath, streamName, topicName }) +console.log("running with params:", { filepath, streamName, topicName }); try { await fileToTopic(filepath, streamName, topicName, 512 * 1024); // eslint-disable-next-line @typescript-eslint/no-unused-vars -} catch(err) { +} catch (err) { process.exit(1); } diff --git a/foreign/node/src/index.ts b/foreign/node/src/index.ts index 687e4c5fd0..1018e98f40 100644 --- a/foreign/node/src/index.ts +++ b/foreign/node/src/index.ts @@ -17,14 +17,14 @@ * under the License. */ - export { type Id, PollingStrategy, Consumer, Partitioning, HeaderValue, -} from './wire/index.js'; + HeaderKeyFactory, +} from "./wire/index.js"; -export * from './client/index.js'; -export * from './stream/index.js'; +export * from "./client/index.js"; +export * from "./stream/index.js"; diff --git a/foreign/node/src/tcp.sm.utils.ts b/foreign/node/src/tcp.sm.utils.ts index ec55d927ee..ed5a8542ad 100644 --- a/foreign/node/src/tcp.sm.utils.ts +++ b/foreign/node/src/tcp.sm.utils.ts @@ -17,64 +17,92 @@ * under the License. */ +import assert from "node:assert/strict"; +import { v7 } from "./wire/uuid.utils.js"; +import { + sendMessages, + type Partitioning, + HeaderValue, + HeaderKeyFactory, + type Message, +} from "./wire/index.js"; +import type { ClientProvider } from "./client/client.type.js"; +import type { Id } from "./wire/identifier.utils.js"; -import assert from 'node:assert/strict'; -import { v7 } from './wire/uuid.utils.js'; -import { sendMessages, type Partitioning, HeaderValue, type Message } from './wire/index.js'; -import type { ClientProvider } from './client/client.type.js'; -import type { Id } from './wire/identifier.utils.js'; - - -const h0 = { 'foo': HeaderValue.Int32(42), 'bar': HeaderValue.Uint8(123) }; -const h1 = { 'x-header-string-1': HeaderValue.String('incredible') }; -const h2 = { 'x-header-bool': HeaderValue.Bool(false) }; +const h0 = [ + { key: HeaderKeyFactory.String("foo"), value: HeaderValue.Int32(42) }, + { key: HeaderKeyFactory.String("bar"), value: HeaderValue.Uint8(123) }, +]; +const h1 = [ + { + key: HeaderKeyFactory.String("x-header-string-1"), + value: HeaderValue.String("incredible"), + }, +]; +const h2 = [ + { + key: HeaderKeyFactory.String("x-header-bool"), + value: HeaderValue.Bool(false), + }, +]; const messages = [ - { payload: 'content with header', headers: h0 }, - { payload: 'content solo' }, - { payload: 'yolo msg' }, - { payload: 'yolo msg 2' }, - { payload: 'this is fuu', headers: h1 }, - { payload: 'this is bar', headers: h2 }, - { payload: 'yolo msg 3' }, - { payload: 'fuu again', headers: h1 }, - { payload: 'damnit', headers: h0 }, - { payload: 'yolo msg 4', Headers: h2 }, + { payload: "content with header", headers: h0 }, + { payload: "content solo" }, + { payload: "yolo msg" }, + { payload: "yolo msg 2" }, + { payload: "this is fuu", headers: h1 }, + { payload: "this is bar", headers: h2 }, + { payload: "yolo msg 3" }, + { payload: "fuu again", headers: h1 }, + { payload: "damnit", headers: h0 }, + { payload: "yolo msg 4", headers: h2 }, ]; -export const someMessageContent = () => messages[Math.floor(Math.random() * messages.length)] +export const someMessageContent = () => + messages[Math.floor(Math.random() * messages.length)]; export const generateMessages = (count = 1) => { return [...Array(count)].map(() => ({ id: v7(), ...someMessageContent() })); -} +}; -export const sendSomeMessages = (s: ClientProvider) => +export const sendSomeMessages = + (s: ClientProvider) => async (streamId: Id, topicId: Id, partition: Partitioning) => { const rSend = await sendMessages(s)({ - topicId, streamId, messages: generateMessages(100), partition + topicId, + streamId, + messages: generateMessages(100), + partition, }); assert.ok(rSend); return rSend; }; - export const formatPolledMessages = (msgs: Message[]) => - msgs.map(m => { - const { headers: { id, offset, timestamp, checksum }, payload, userHeaders } = m; + msgs.map((m) => { + const { + headers: { id, offset, timestamp, checksum }, + payload, + userHeaders, + } = m; return { id, offset, headers: userHeaders, payload: payload.toString(), timestamp, - checksum + checksum, }; }); -export const getIggyAddress = (host = '127.0.0.1', port = 8090): [string, number] => { +export const getIggyAddress = ( + host = "127.0.0.1", + port = 8090, +): [string, number] => { if (process.env.IGGY_TCP_ADDRESS) { - const s = (process.env.IGGY_TCP_ADDRESS || '').split(':'); + const s = (process.env.IGGY_TCP_ADDRESS || "").split(":"); [host, port] = [s[0], s[1] ? parseInt(s[1].toString(), 10) : port]; } return [host, port]; -} +}; diff --git a/foreign/node/src/wire/message/header.type.ts b/foreign/node/src/wire/message/header.type.ts index 602e514502..a99f676c9a 100644 --- a/foreign/node/src/wire/message/header.type.ts +++ b/foreign/node/src/wire/message/header.type.ts @@ -38,7 +38,7 @@ export const HeaderKind = { Uint64: 12, Uint128: 13, Float: 14, - Double: 15 + Double: 15, } as const; /** Type alias for the HeaderKind object */ @@ -50,95 +50,121 @@ export type HeaderKindValue = ValueOf; /** Reverse mapping from numeric value to header kind name */ export const ReverseHeaderKind = reverseRecord(HeaderKind); +/** Returns expected byte size for a header kind, or -1 for variable-size kinds */ +export const expectedSize = (kind: number): number => { + switch (kind) { + case HeaderKind.Bool: + case HeaderKind.Int8: + case HeaderKind.Uint8: + return 1; + case HeaderKind.Int16: + case HeaderKind.Uint16: + return 2; + case HeaderKind.Int32: + case HeaderKind.Uint32: + case HeaderKind.Float: + return 4; + case HeaderKind.Int64: + case HeaderKind.Uint64: + case HeaderKind.Double: + return 8; + case HeaderKind.Int128: + case HeaderKind.Uint128: + return 16; + default: + return -1; + } +}; + /** Raw binary header value */ export type HeaderValueRaw = { - kind: HeaderKind['Raw'], - value: Buffer -} + kind: HeaderKind["Raw"]; + value: Buffer; +}; /** String header value */ export type HeaderValueString = { - kind: HeaderKind['String'] - value: string -} + kind: HeaderKind["String"]; + value: string; +}; /** Boolean header value */ export type HeaderValueBool = { - kind: HeaderKind['Bool'], - value: boolean -} + kind: HeaderKind["Bool"]; + value: boolean; +}; /** Signed 8-bit integer header value */ export type HeaderValueInt8 = { - kind: HeaderKind['Int8'], - value: number -} + kind: HeaderKind["Int8"]; + value: number; +}; /** Signed 16-bit integer header value */ export type HeaderValueInt16 = { - kind: HeaderKind['Int16'], - value: number + kind: HeaderKind["Int16"]; + value: number; }; /** Signed 32-bit integer header value */ export type HeaderValueInt32 = { - kind: HeaderKind['Int32'], - value: number -} + kind: HeaderKind["Int32"]; + value: number; +}; /** Signed 64-bit integer header value */ export type HeaderValueInt64 = { - kind: HeaderKind['Int64'], - value: bigint -} + kind: HeaderKind["Int64"]; + value: bigint; +}; /** Signed 128-bit integer header value */ export type HeaderValueInt128 = { - kind: HeaderKind['Int128'], - value: Buffer // | ArrayBuffer // ? -} + kind: HeaderKind["Int128"]; + value: Buffer; // | ArrayBuffer // ? +}; /** Unsigned 8-bit integer header value */ export type HeaderValueUint8 = { - kind: HeaderKind['Uint8'], - value: number -} + kind: HeaderKind["Uint8"]; + value: number; +}; /** Unsigned 16-bit integer header value */ export type HeaderValueUint16 = { - kind: HeaderKind['Uint16'], - value: number -} + kind: HeaderKind["Uint16"]; + value: number; +}; /** Unsigned 32-bit integer header value */ export type HeaderValueUint32 = { - kind: HeaderKind['Uint32'], - value: number -} + kind: HeaderKind["Uint32"]; + value: number; +}; /** Unsigned 64-bit integer header value */ export type HeaderValueUint64 = { - kind: HeaderKind['Uint64'], - value: bigint -} + kind: HeaderKind["Uint64"]; + value: bigint; +}; /** Unsigned 128-bit integer header value */ export type HeaderValueUint128 = { - kind: HeaderKind['Uint128'], - value: Buffer // | ArrayBuffer // ? -} + kind: HeaderKind["Uint128"]; + value: Buffer; // | ArrayBuffer // ? +}; /** 32-bit floating point header value */ export type HeaderValueFloat = { - kind: HeaderKind['Float'], - value: number -} + kind: HeaderKind["Float"]; + value: number; +}; /** 64-bit floating point (double) header value */ export type HeaderValueDouble = { - kind: HeaderKind['Double'], - value: number -} + kind: HeaderKind["Double"]; + value: number; +}; // export type HeaderValue = // HeaderValueRaw | @@ -158,3 +184,24 @@ export type HeaderValueDouble = { // HeaderValueDouble; // export type Headers = Record; + +/** Header key with raw bytes */ +export type HeaderKeyRaw = { + kind: HeaderKind["Raw"]; + value: Buffer; +}; + +/** Header key with string value */ +export type HeaderKeyString = { + kind: HeaderKind["String"]; + value: string; +}; + +/** Header key with Int32 value */ +export type HeaderKeyInt32 = { + kind: HeaderKind["Int32"]; + value: number; +}; + +/** Union type of all possible header key types */ +export type HeaderKey = HeaderKeyRaw | HeaderKeyString | HeaderKeyInt32; diff --git a/foreign/node/src/wire/message/header.utils.test.ts b/foreign/node/src/wire/message/header.utils.test.ts index 598b0d8d8c..dedb3bcd0b 100644 --- a/foreign/node/src/wire/message/header.utils.test.ts +++ b/foreign/node/src/wire/message/header.utils.test.ts @@ -17,28 +17,54 @@ * under the License. */ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { uuidv7, uuidv4 } from "uuidv7"; +import { + serializeHeaders, + deserializeHeaders, + HeaderValue, + HeaderKeyFactory, +} from "./header.utils.js"; +import { HeaderKind } from "./header.type.js"; -import { describe, it } from 'node:test'; -import assert from 'node:assert/strict'; -import { uuidv7, uuidv4 } from 'uuidv7' -import { serializeHeaders, deserializeHeaders, HeaderValue } from './header.utils.js'; +describe("Headers", () => { + const headers = [ + { key: HeaderKeyFactory.String("p"), value: HeaderValue.Bool(true) }, + { key: HeaderKeyFactory.String("x"), value: HeaderValue.Uint32(123) }, + { key: HeaderKeyFactory.String("y"), value: HeaderValue.Uint64(42n) }, + { + key: HeaderKeyFactory.String("z"), + value: HeaderValue.Float(42.20000076293945), + }, + { key: HeaderKeyFactory.String("a"), value: HeaderValue.Double(1 / 3) }, + { key: HeaderKeyFactory.String("ID"), value: HeaderValue.String(uuidv7()) }, + { + key: HeaderKeyFactory.String("val"), + value: HeaderValue.Raw(Buffer.from(uuidv4())), + }, + ]; -describe('Headers', () => { - - const headers = { - 'p': HeaderValue.Bool(true), - 'x': HeaderValue.Uint32(123), - 'y': HeaderValue.Uint64(42n), - 'z': HeaderValue.Float(42.20000076293945), - 'a': HeaderValue.Double(1/3), - 'ID': HeaderValue.String(uuidv7()), - 'val': HeaderValue.Raw(Buffer.from(uuidv4())) - }; - - it('serialize/deserialize', () => { + it("serialize/deserialize string keys", () => { const s = serializeHeaders(headers); const d = deserializeHeaders(s); - assert.deepEqual(headers, d); + assert.equal(d.length, headers.length); + for (let i = 0; i < headers.length; i++) { + assert.equal(d[i].key.kind, headers[i].key.kind); + assert.equal(d[i].value.kind, headers[i].value.kind); + } }); + it("serialize/deserialize int32 key", () => { + const int32Headers = [ + { key: HeaderKeyFactory.Int32(42), value: HeaderValue.String("test") }, + ]; + const s = serializeHeaders(int32Headers); + const d = deserializeHeaders(s); + assert.equal(d.length, 1); + assert.equal(d[0].key.kind, HeaderKind.Int32); + assert.equal(d[0].key.value, 42); + assert.equal(d[0].value.kind, HeaderKind.String); + assert.equal(d[0].value.value, "test"); + }); }); diff --git a/foreign/node/src/wire/message/header.utils.ts b/foreign/node/src/wire/message/header.utils.ts index fa6e9f57db..320d107013 100644 --- a/foreign/node/src/wire/message/header.utils.ts +++ b/foreign/node/src/wire/message/header.utils.ts @@ -17,7 +17,6 @@ * under the License. */ - import { boolToBuf, int8ToBuf, @@ -29,8 +28,8 @@ import { uint32ToBuf, uint64ToBuf, floatToBuf, - doubleToBuf -} from '../number.utils.js'; + doubleToBuf, +} from "../number.utils.js"; import { type HeaderValueRaw, @@ -50,46 +49,53 @@ import { type HeaderValueDouble, type HeaderKindId, type HeaderKindValue, + type HeaderKey, + type HeaderKeyRaw, + type HeaderKeyString, + type HeaderKeyInt32, HeaderKind, - ReverseHeaderKind -} from './header.type.js'; - + ReverseHeaderKind, + expectedSize, +} from "./header.type.js"; /** * Union type of all possible header value types. */ export type HeaderValue = - HeaderValueRaw | - HeaderValueString | - HeaderValueBool | - HeaderValueInt8 | - HeaderValueInt16 | - HeaderValueInt32 | - HeaderValueInt64 | - HeaderValueInt128 | - HeaderValueUint8 | - HeaderValueUint16 | - HeaderValueUint32 | - HeaderValueUint64 | - HeaderValueUint128 | - HeaderValueFloat | - HeaderValueDouble; - -/** - * Map of header names to header values. - */ -export type Headers = Record; + | HeaderValueRaw + | HeaderValueString + | HeaderValueBool + | HeaderValueInt8 + | HeaderValueInt16 + | HeaderValueInt32 + | HeaderValueInt64 + | HeaderValueInt128 + | HeaderValueUint8 + | HeaderValueUint16 + | HeaderValueUint32 + | HeaderValueUint64 + | HeaderValueUint128 + | HeaderValueFloat + | HeaderValueDouble; + +/** Header entry with key and value */ +export type HeaderEntry = { + key: HeaderKey; + value: HeaderValue; +}; +/** Array of header entries */ +export type Headers = HeaderEntry[]; /** * Internal representation of a header value in binary format. */ type BinaryHeaderValue = { /** Header kind identifier */ - kind: number,// HeaderKind, + kind: number; // HeaderKind, /** Serialized value as Buffer */ - value: Buffer -} + value: Buffer; +}; /** * Serializes a header value to a Buffer based on its kind. @@ -100,56 +106,76 @@ type BinaryHeaderValue = { export const serializeHeaderValue = (header: HeaderValue) => { const { kind, value } = header; switch (kind) { - case HeaderKind.Raw: return value; - case HeaderKind.String: return Buffer.from(value); - case HeaderKind.Bool: return boolToBuf(value); - case HeaderKind.Int8: return int8ToBuf(value); - case HeaderKind.Int16: return int16ToBuf(value); - case HeaderKind.Int32: return int32ToBuf(value); - case HeaderKind.Int64: return int64ToBuf(value); - case HeaderKind.Int128: return value; - case HeaderKind.Uint8: return uint8ToBuf(value); - case HeaderKind.Uint16: return uint16ToBuf(value); - case HeaderKind.Uint32: return uint32ToBuf(value); - case HeaderKind.Uint64: return uint64ToBuf(value); - case HeaderKind.Uint128: return value; - case HeaderKind.Float: return floatToBuf(value); - case HeaderKind.Double: return doubleToBuf(value); + case HeaderKind.Raw: + return value; + case HeaderKind.String: + return Buffer.from(value); + case HeaderKind.Bool: + return boolToBuf(value); + case HeaderKind.Int8: + return int8ToBuf(value); + case HeaderKind.Int16: + return int16ToBuf(value); + case HeaderKind.Int32: + return int32ToBuf(value); + case HeaderKind.Int64: + return int64ToBuf(value); + case HeaderKind.Int128: + return value; + case HeaderKind.Uint8: + return uint8ToBuf(value); + case HeaderKind.Uint16: + return uint16ToBuf(value); + case HeaderKind.Uint32: + return uint32ToBuf(value); + case HeaderKind.Uint64: + return uint64ToBuf(value); + case HeaderKind.Uint128: + return value; + case HeaderKind.Float: + return floatToBuf(value); + case HeaderKind.Double: + return doubleToBuf(value); } }; +/** + * Serializes a header key to a Buffer based on its kind. + * + * @param key - Header key to serialize + * @returns Serialized key as Buffer + */ +export const serializeHeaderKey = (key: HeaderKey): Buffer => { + const { kind, value } = key; + switch (kind) { + case HeaderKind.Raw: + return value; + case HeaderKind.String: + return Buffer.from(value); + case HeaderKind.Int32: + return int32ToBuf(value); + } +}; /** * Serializes a single header key-value pair to wire format. - * Format: [key_length][key][kind][value_length][value] + * Format: [key_kind][key_length][key][value_kind][value_length][value] * - * @param key - Header key name + * @param key - Header key * @param v - Binary header value * @returns Serialized header as Buffer */ -export const serializeHeader = (key: string, v: BinaryHeaderValue) => { - const bKey = Buffer.from(key) - const b1 = uint32ToBuf(bKey.length); - const b2 = Buffer.alloc(5); - b2.writeUInt8(v.kind); - b2.writeUInt32LE(v.value.length, 1); - - // @TODO debug - // console.log( - // 'SERIALIZE\n', - // 'KEY-LEN', b1.length, b1.toString('hex'), '=', b1.readUInt32LE(0), '\n', - // 'KEY', bKey.length, bKey.toString('hex'), '\n', - // 'KIND', b2.readUInt8(0), b2.subarray(0, 1).toString('hex'), '\n', - // 'V-LEN', b2.readUInt32LE(1), b2.subarray(1, 5).toString('hex'), '\n', - // 'V', v.value.length, v.value.toString('hex') - // ); - - return Buffer.concat([ - b1, - bKey, - b2, - v.value - ]); +export const serializeHeader = (key: HeaderKey, v: BinaryHeaderValue) => { + const bKey = serializeHeaderKey(key); + const keyHeader = Buffer.alloc(5); + keyHeader.writeUInt8(key.kind); + keyHeader.writeUInt32LE(bKey.length, 1); + + const valueHeader = Buffer.alloc(5); + valueHeader.writeUInt8(v.kind); + valueHeader.writeUInt32LE(v.value.length, 1); + + return Buffer.concat([keyHeader, bKey, valueHeader, v.value]); }; /** Empty headers buffer constant */ @@ -163,22 +189,23 @@ export const EMPTY_HEADERS = Buffer.alloc(0); */ const createHeaderValue = (header: HeaderValue): BinaryHeaderValue => ({ kind: header.kind, - value: serializeHeaderValue(header) + value: serializeHeaderValue(header), }); /** * Serializes all headers to a single buffer. * - * @param headers - Optional headers map + * @param headers - Optional headers array * @returns Serialized headers buffer (empty if no headers) */ export const serializeHeaders = (headers?: Headers) => { - if (!headers) - return EMPTY_HEADERS; + if (!headers || headers.length === 0) return EMPTY_HEADERS; - return Buffer.concat(Object.keys(headers).map( - (c: string) => serializeHeader(c, createHeaderValue(headers[c])) - )); + return Buffer.concat( + headers.map((entry) => + serializeHeader(entry.key, createHeaderValue(entry.value)), + ), + ); }; // deserialize ... @@ -186,31 +213,33 @@ export const serializeHeaders = (headers?: Headers) => { /** Possible JavaScript types for deserialized header values */ export type ParsedHeaderValue = boolean | string | number | bigint | Buffer; -/** - * Deserialized header with kind and value. - */ -export type ParsedHeader = { - /** Header kind identifier */ - kind: ParsedHeaderValue, - /** Deserialized value */ - value: ParsedHeaderValue -} +/** Deserialized header key with kind and value */ +export type ParsedHeaderKey = { + kind: number; + value: ParsedHeaderValue; +}; -/** Header with its key included */ -type HeaderWithKey = ParsedHeader & { key: string }; +/** Deserialized header value with kind and value */ +export type ParsedHeaderVal = { + kind: number; + value: ParsedHeaderValue; +}; -/** Map of header keys to parsed headers */ -export type HeadersMap = Record; +/** Deserialized header entry */ +export type ParsedHeaderEntry = { + key: ParsedHeaderKey; + value: ParsedHeaderVal; +}; /** * Result of deserializing a single header. */ type ParsedHeaderDeserialized = { /** Number of bytes consumed */ - bytesRead: number, - /** Deserialized header data with key */ - data: HeaderWithKey -} + bytesRead: number; + /** Deserialized header entry */ + data: ParsedHeaderEntry; +}; /** * Maps a numeric header kind to its string identifier. @@ -223,7 +252,7 @@ export const mapHeaderKind = (k: number): HeaderKindId => { if (!ReverseHeaderKind[k as HeaderKindValue]) throw new Error(`unknow header kind: ${k}`); return ReverseHeaderKind[k as HeaderKindValue]; -} +}; /** * Deserializes a header value buffer based on its kind. @@ -233,72 +262,120 @@ export const mapHeaderKind = (k: number): HeaderKindId => { * @returns Deserialized value * @throws Error if the header kind is invalid */ -export const deserializeHeaderValue = - (kind: number, value: Buffer): ParsedHeaderValue => { - switch (kind) { - case HeaderKind.Int128: - case HeaderKind.Uint128: - case HeaderKind.Raw: return value; - case HeaderKind.String: return value.toString(); - case HeaderKind.Int8: return value.readInt8(); - case HeaderKind.Int16: return value.readInt16LE(); - case HeaderKind.Int32: return value.readInt32LE(); - case HeaderKind.Int64: return value.readBigInt64LE(); - case HeaderKind.Uint8: return value.readUint8(); - case HeaderKind.Uint16: return value.readUint16LE(); - case HeaderKind.Uint32: return value.readUInt32LE(); - case HeaderKind.Uint64: return value.readBigUInt64LE(); - case HeaderKind.Bool: return value.readUInt8() === 1; - case HeaderKind.Float: return value.readFloatLE(); - case HeaderKind.Double: return value.readDoubleLE(); - default: throw new Error(`deserializeHeaderValue: invalid HeaderKind ${kind}`); - } - }; +export const deserializeHeaderValue = ( + kind: number, + value: Buffer, +): ParsedHeaderValue => { + switch (kind) { + case HeaderKind.Int128: + case HeaderKind.Uint128: + case HeaderKind.Raw: + return value; + case HeaderKind.String: + return value.toString(); + case HeaderKind.Int8: + return value.readInt8(); + case HeaderKind.Int16: + return value.readInt16LE(); + case HeaderKind.Int32: + return value.readInt32LE(); + case HeaderKind.Int64: + return value.readBigInt64LE(); + case HeaderKind.Uint8: + return value.readUint8(); + case HeaderKind.Uint16: + return value.readUint16LE(); + case HeaderKind.Uint32: + return value.readUInt32LE(); + case HeaderKind.Uint64: + return value.readBigUInt64LE(); + case HeaderKind.Bool: + return value.readUInt8() === 1; + case HeaderKind.Float: + return value.readFloatLE(); + case HeaderKind.Double: + return value.readDoubleLE(); + default: + throw new Error(`deserializeHeaderValue: invalid HeaderKind ${kind}`); + } +}; /** * Deserializes a single header from a buffer. + * Format: [key_kind][key_length][key][value_kind][value_length][value] * * @param p - Buffer containing serialized headers * @param pos - Starting position in the buffer * @returns Object with bytes read and deserialized header data + * @throws Error if header key or value length is invalid (must be 1-255) */ -export const deserializeHeader = (p: Buffer, pos = 0): ParsedHeaderDeserialized => { - const keyLength = p.readUInt32LE(pos); - const key = p.subarray(pos + 4, pos + 4 + keyLength).toString(); - pos += keyLength + 4; - const rawKind = p.readUInt8(pos); - // @TODO ? - // const kind = mapHeaderKind(rawKind); +export const deserializeHeader = ( + p: Buffer, + pos = 0, +): ParsedHeaderDeserialized => { + const keyKind = p.readUInt8(pos); + const keyLength = p.readUInt32LE(pos + 1); + if (keyLength < 1 || keyLength > 255) { + throw new Error( + `Invalid header key length: ${keyLength}, must be between 1 and 255`, + ); + } + const keyExpected = expectedSize(keyKind); + if (keyExpected !== -1 && keyLength !== keyExpected) { + throw new Error( + `Invalid header key size for kind ${keyKind}: expected ${keyExpected}, got ${keyLength}`, + ); + } + const keyValue = deserializeHeaderValue( + keyKind, + p.subarray(pos + 5, pos + 5 + keyLength), + ); + pos += 5 + keyLength; + + const valueKind = p.readUInt8(pos); const valueLength = p.readUInt32LE(pos + 1); - const value = deserializeHeaderValue(rawKind, p.subarray(pos + 5, pos + 5 + valueLength)); + if (valueLength < 1 || valueLength > 255) { + throw new Error( + `Invalid header value length: ${valueLength}, must be between 1 and 255`, + ); + } + const valueExpected = expectedSize(valueKind); + if (valueExpected !== -1 && valueLength !== valueExpected) { + throw new Error( + `Invalid header value size for kind ${valueKind}: expected ${valueExpected}, got ${valueLength}`, + ); + } + const value = deserializeHeaderValue( + valueKind, + p.subarray(pos + 5, pos + 5 + valueLength), + ); return { - bytesRead: 4 + 4 + 1 + keyLength + valueLength, + bytesRead: 5 + keyLength + 5 + valueLength, data: { - key, - kind: rawKind, - value - } + key: { kind: keyKind, value: keyValue }, + value: { kind: valueKind, value }, + }, }; -} +}; /** * Deserializes all headers from a buffer. * * @param p - Buffer containing serialized headers * @param pos - Starting position in the buffer - * @returns Map of header keys to parsed headers + * @returns Array of parsed header entries */ -export const deserializeHeaders = (p: Buffer, pos = 0) => { - const headers: HeadersMap = {}; +export const deserializeHeaders = (p: Buffer, pos = 0): ParsedHeaderEntry[] => { + const headers: ParsedHeaderEntry[] = []; const len = p.length; while (pos < len) { - const { bytesRead, data: { kind, key, value } } = deserializeHeader(p, pos); - headers[key] = { kind, value }; + const { bytesRead, data } = deserializeHeader(p, pos); + headers.push(data); pos += bytesRead; } return headers; -} +}; /** * HeaderValue factory functions and utilities. @@ -308,91 +385,91 @@ export const deserializeHeaders = (p: Buffer, pos = 0) => { /** Creates a raw binary header value */ const Raw = (value: Buffer): HeaderValueRaw => ({ kind: HeaderKind.Raw, - value + value, }); /** Creates a string header value */ const String = (value: string): HeaderValueString => ({ kind: HeaderKind.String, - value + value, }); /** Creates a boolean header value */ const Bool = (value: boolean): HeaderValueBool => ({ kind: HeaderKind.Bool, - value + value, }); /** Creates an Int8 header value */ const Int8 = (value: number): HeaderValueInt8 => ({ kind: HeaderKind.Int8, - value + value, }); /** Creates an Int16 header value */ const Int16 = (value: number): HeaderValueInt16 => ({ kind: HeaderKind.Int16, - value + value, }); /** Creates an Int32 header value */ const Int32 = (value: number): HeaderValueInt32 => ({ kind: HeaderKind.Int32, - value + value, }); /** Creates an Int64 header value */ const Int64 = (value: bigint): HeaderValueInt64 => ({ kind: HeaderKind.Int64, - value + value, }); /** Creates an Int128 header value */ const Int128 = (value: Buffer): HeaderValueInt128 => ({ kind: HeaderKind.Int128, - value + value, }); /** Creates a Uint8 header value */ const Uint8 = (value: number): HeaderValueUint8 => ({ kind: HeaderKind.Uint8, - value + value, }); /** Creates a Uint16 header value */ const Uint16 = (value: number): HeaderValueUint16 => ({ kind: HeaderKind.Uint16, - value + value, }); /** Creates a Uint32 header value */ const Uint32 = (value: number): HeaderValueUint32 => ({ kind: HeaderKind.Uint32, - value + value, }); /** Creates a Uint64 header value */ const Uint64 = (value: bigint): HeaderValueUint64 => ({ kind: HeaderKind.Uint64, - value + value, }); /** Creates a Uint128 header value */ const Uint128 = (value: Buffer): HeaderValueUint128 => ({ kind: HeaderKind.Uint128, - value + value, }); /** Creates a Float header value */ const Float = (value: number): HeaderValueFloat => ({ kind: HeaderKind.Float, - value + value, }); /** Creates a Double header value */ const Double = (value: number): HeaderValueDouble => ({ kind: HeaderKind.Double, - value + value, }); /** Gets the kind identifier string of a header value */ @@ -420,10 +497,32 @@ export const HeaderValue = { Float, Double, getKind, - getValue + getValue, }; +/** Creates a raw binary header key */ +const keyRaw = (value: Buffer): HeaderKeyRaw => ({ + kind: HeaderKind.Raw, + value, +}); +/** Creates a string header key */ +const keyString = (value: string): HeaderKeyString => ({ + kind: HeaderKind.String, + value, +}); + +/** Creates an Int32 header key */ +const keyInt32 = (value: number): HeaderKeyInt32 => ({ + kind: HeaderKind.Int32, + value, +}); + +export const HeaderKeyFactory = { + Raw: keyRaw, + String: keyString, + Int32: keyInt32, +}; // export type InputHeaderValue = boolean | number | string | bigint | Buffer; // export type InputHeaders = Record; @@ -457,7 +556,6 @@ export const HeaderValue = { // export const createHeaderValueString = (v: string): HeaderValue => // ({ kind: HeaderKind.String, value: Buffer.from(v) }); - // // guess wire type from js type ? // const guessHeaderValue = (v: InputHeaderValue): HeaderValue => { // if (typeof v === 'number') { diff --git a/foreign/node/src/wire/message/index.ts b/foreign/node/src/wire/message/index.ts index f5e01049c8..903645d220 100644 --- a/foreign/node/src/wire/message/index.ts +++ b/foreign/node/src/wire/message/index.ts @@ -17,11 +17,10 @@ * under the License. */ - -export * from './poll-messages.command.js'; -export * from './send-messages.command.js'; -export * from './message.utils.js'; +export * from "./poll-messages.command.js"; +export * from "./send-messages.command.js"; +export * from "./message.utils.js"; export * from "./poll.utils.js"; -export { Partitioning } from './partitioning.utils.js'; -export { HeaderValue } from './header.utils.js'; +export { Partitioning } from "./partitioning.utils.js"; +export { HeaderValue, HeaderKeyFactory } from "./header.utils.js"; diff --git a/foreign/node/src/wire/message/poll.utils.ts b/foreign/node/src/wire/message/poll.utils.ts index fbdaa244d8..34fce59b2a 100644 --- a/foreign/node/src/wire/message/poll.utils.ts +++ b/foreign/node/src/wire/message/poll.utils.ts @@ -17,17 +17,16 @@ * under the License. */ - -import { type Id } from '../identifier.utils.js'; -import { type ValueOf, reverseRecord } from '../../type.utils.js'; -import { serializeGetOffset, type Consumer } from '../offset/offset.utils.js'; -import { deserializeHeaders, type HeadersMap } from './header.utils.js'; -import { Transform, type TransformCallback } from 'node:stream'; +import { type Id } from "../identifier.utils.js"; +import { type ValueOf, reverseRecord } from "../../type.utils.js"; +import { serializeGetOffset, type Consumer } from "../offset/offset.utils.js"; +import { deserializeHeaders, type ParsedHeaderEntry } from "./header.utils.js"; +import { Transform, type TransformCallback } from "node:stream"; import { deserializeIggyMessageHeaders, IGGY_MESSAGE_HEADER_SIZE, - type IggyMessageHeader -} from './iggy-header.utils.js'; + type IggyMessageHeader, +} from "./iggy-header.utils.js"; /** * Enumeration of message polling strategies. @@ -42,7 +41,7 @@ export const PollingStrategyKind = { /** Poll from the last message */ Last: 4, /** Poll the next unconsumed message */ - Next: 5 + Next: 5, } as const; /** Type alias for the PollingStrategyKind object */ @@ -50,65 +49,64 @@ export type PollingStrategyKind = typeof PollingStrategyKind; /** String literal type of polling strategy names */ export type PollingStrategyKindId = keyof PollingStrategyKind; /** Numeric values of polling strategies */ -export type PollingStrategyKindValue = ValueOf +export type PollingStrategyKindValue = ValueOf; /** Polling from a specific offset */ export type OffsetPollingStrategy = { - kind: PollingStrategyKind['Offset'], + kind: PollingStrategyKind["Offset"]; /** Offset to start polling from */ - value: bigint -} + value: bigint; +}; /** Polling from a specific timestamp */ export type TimestampPollingStrategy = { - kind: PollingStrategyKind['Timestamp'], + kind: PollingStrategyKind["Timestamp"]; /** Timestamp in microseconds */ - value: bigint -} + value: bigint; +}; /** Polling from the first message */ export type FirstPollingStrategy = { - kind: PollingStrategyKind['First'], - value: 0n -} + kind: PollingStrategyKind["First"]; + value: 0n; +}; /** Polling from the last message */ export type LastPollingStrategy = { - kind: PollingStrategyKind['Last'], - value: 0n -} + kind: PollingStrategyKind["Last"]; + value: 0n; +}; /** Polling the next unconsumed message */ export type NextPollingStrategy = { - kind: PollingStrategyKind['Next'], - value: 0n -} + kind: PollingStrategyKind["Next"]; + value: 0n; +}; /** Union of all polling strategy types */ export type PollingStrategy = - OffsetPollingStrategy | - TimestampPollingStrategy | - FirstPollingStrategy | - LastPollingStrategy | - NextPollingStrategy; - + | OffsetPollingStrategy + | TimestampPollingStrategy + | FirstPollingStrategy + | LastPollingStrategy + | NextPollingStrategy; /** Next polling strategy constant */ const Next: NextPollingStrategy = { kind: PollingStrategyKind.Next, - value:0n + value: 0n, }; /** First polling strategy constant */ const First: FirstPollingStrategy = { kind: PollingStrategyKind.First, - value:0n + value: 0n, }; /** Last polling strategy constant */ const Last: LastPollingStrategy = { kind: PollingStrategyKind.Last, - value:0n + value: 0n, }; /** @@ -119,7 +117,7 @@ const Last: LastPollingStrategy = { */ const Offset = (n: bigint): OffsetPollingStrategy => ({ kind: PollingStrategyKind.Offset, - value: n + value: n, }); /** @@ -130,7 +128,7 @@ const Offset = (n: bigint): OffsetPollingStrategy => ({ */ const Timestamp = (n: bigint): TimestampPollingStrategy => ({ kind: PollingStrategyKind.Timestamp, - value: n + value: n, }); /** @@ -141,10 +139,9 @@ export const PollingStrategy = { First, Last, Offset, - Timestamp + Timestamp, }; - /** * Serializes a poll messages command payload. * @@ -174,7 +171,7 @@ export const serializePollMessages = ( return Buffer.concat([ serializeGetOffset(streamId, topicId, consumer, partitionId), - b + b, ]); }; @@ -189,8 +186,8 @@ export const MessageState = { /** Message processing failed */ Poisoned: 20, /** Message is scheduled for deletion */ - MarkedForDeletion: 30 -} + MarkedForDeletion: 30, +}; /** Type alias for the MessageState object */ type MessageState = typeof MessageState; @@ -209,21 +206,21 @@ const ReverseMessageState = reverseRecord(MessageState); * @throws Error if the state is unknown */ export const mapMessageState = (k: number): MessageStateId => { - if(!ReverseMessageState[k as MessageStateValue]) + if (!ReverseMessageState[k as MessageStateValue]) throw new Error(`unknow message state: ${k}`); return ReverseMessageState[k as MessageStateValue]; -} +}; /** * A polled message with headers, payload, and user headers. */ export type Message = { /** Iggy message header metadata */ - headers: IggyMessageHeader, + headers: IggyMessageHeader; /** Message payload data */ - payload: Buffer, + payload: Buffer; /** User-defined headers */ - userHeaders: HeadersMap + userHeaders: ParsedHeaderEntry[]; }; /** @@ -231,13 +228,13 @@ export type Message = { */ export type PollMessagesResponse = { /** Partition the messages came from */ - partitionId: number, + partitionId: number; /** Current offset in the partition */ - currentOffset: bigint, + currentOffset: bigint; /** Number of messages returned */ - count: number, + count: number; /** Array of polled messages */ - messages: Message[] + messages: Message[]; }; /** @@ -251,29 +248,32 @@ export const deserializeMessages = (b: Buffer) => { let pos = 0; const len = b.length; while (pos < len) { - if(pos + IGGY_MESSAGE_HEADER_SIZE > len) - break; + if (pos + IGGY_MESSAGE_HEADER_SIZE > len) break; const bHead = b.subarray(pos, pos + IGGY_MESSAGE_HEADER_SIZE); const headers = deserializeIggyMessageHeaders(bHead); pos += IGGY_MESSAGE_HEADER_SIZE; const plEnd = pos + headers.payloadLength; - if(plEnd > len) - break; + if (plEnd > len) break; const payload = b.subarray(pos, plEnd); pos += headers.payloadLength; - let userHeaders: HeadersMap = {}; - if(headers.userHeadersLength > 0 && plEnd + headers.userHeadersLength <= len) { - userHeaders = deserializeHeaders(b.subarray(plEnd, plEnd + headers.userHeadersLength)); + let userHeaders: ParsedHeaderEntry[] = []; + if ( + headers.userHeadersLength > 0 && + plEnd + headers.userHeadersLength <= len + ) { + userHeaders = deserializeHeaders( + b.subarray(plEnd, plEnd + headers.userHeadersLength), + ); pos += headers.userHeadersLength; } messages.push({ headers, payload, - userHeaders + userHeaders, }); } return messages; -} +}; /** * Deserializes a poll messages response from a buffer. @@ -292,8 +292,8 @@ export const deserializePollMessages = (r: Buffer, pos = 0) => { partitionId, currentOffset, count, - messages - } + messages, + }; }; /** @@ -301,13 +301,17 @@ export const deserializePollMessages = (r: Buffer, pos = 0) => { * * @returns Transform stream that outputs PollMessagesResponse objects */ -export const deserializePollMessagesTransform = () => new Transform({ - objectMode: true, - transform(chunk: Buffer, encoding: BufferEncoding, cb: TransformCallback) { - try { - return cb(null, deserializePollMessages(chunk)); - } catch (err: unknown) { - cb(new Error('deserializePollMessage::transform error', { cause: err }), null); - } - } -}) +export const deserializePollMessagesTransform = () => + new Transform({ + objectMode: true, + transform(chunk: Buffer, encoding: BufferEncoding, cb: TransformCallback) { + try { + return cb(null, deserializePollMessages(chunk)); + } catch (err: unknown) { + cb( + new Error("deserializePollMessage::transform error", { cause: err }), + null, + ); + } + }, + }); diff --git a/foreign/node/src/wire/message/send-messages.command.test.ts b/foreign/node/src/wire/message/send-messages.command.test.ts index e4945b6578..5538858faa 100644 --- a/foreign/node/src/wire/message/send-messages.command.test.ts +++ b/foreign/node/src/wire/message/send-messages.command.test.ts @@ -17,109 +17,146 @@ * under the License. */ - -import { describe, it } from 'node:test'; -import assert from 'node:assert/strict'; -import { uuidv7, uuidv4 } from 'uuidv7' -import { SEND_MESSAGES, type SendMessages } from './send-messages.command.js'; -import { HeaderValue } from './header.utils.js'; - -describe('SendMessages', () => { - - describe('serialize', () => { - +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { uuidv7, uuidv4 } from "uuidv7"; +import { SEND_MESSAGES, type SendMessages } from "./send-messages.command.js"; +import { HeaderValue, HeaderKeyFactory } from "./header.utils.js"; + +describe("SendMessages", () => { + describe("serialize", () => { const t1 = { streamId: 911, topicId: 213, messages: [ - { payload: 'a' }, - { id: 0, payload: 'b' }, - { id: 123, payload: 'X' }, - { id: 0n, payload: 'c' }, - { id: 1236234534554n, payload: 'X' }, - { id: uuidv4(), payload: 'd' }, - { id: uuidv7(), payload: 'e' }, + { payload: "a" }, + { id: 0, payload: "b" }, + { id: 123, payload: "X" }, + { id: 0n, payload: "c" }, + { id: 1236234534554n, payload: "X" }, + { id: uuidv4(), payload: "d" }, + { id: uuidv7(), payload: "e" }, ], }; - it('serialize SendMessages into a buffer', () => { - assert.deepEqual( - SEND_MESSAGES.serialize(t1).length, - 533 - ); + it("serialize SendMessages into a buffer", () => { + assert.deepEqual(SEND_MESSAGES.serialize(t1).length, 533); }); - it('serialize all kinds of messageId', () => { - assert.doesNotThrow( - () => SEND_MESSAGES.serialize(t1), - ); + it("serialize all kinds of messageId", () => { + assert.doesNotThrow(() => SEND_MESSAGES.serialize(t1)); }); - - it('does not throw on number message id', () => { - const t = { ...t1, messages: [{ id: 42, payload: 'm' }] }; - assert.doesNotThrow( - () => SEND_MESSAGES.serialize(t) - ); + it("does not throw on number message id", () => { + const t = { ...t1, messages: [{ id: 42, payload: "m" }] }; + assert.doesNotThrow(() => SEND_MESSAGES.serialize(t)); }); - it('does not throw on bigint message id', () => { - const t = { ...t1, messages: [{ id: 123n, payload: 'm' }] }; - assert.doesNotThrow( - () => SEND_MESSAGES.serialize(t) - ); + it("does not throw on bigint message id", () => { + const t = { ...t1, messages: [{ id: 123n, payload: "m" }] }; + assert.doesNotThrow(() => SEND_MESSAGES.serialize(t)); }); - it('does not throw on uuid message id', () => { - const t = { ...t1, messages: [{ id: uuidv4(), payload: 'uuid' }] }; - assert.doesNotThrow( - () => SEND_MESSAGES.serialize(t) - ); + it("does not throw on uuid message id", () => { + const t = { ...t1, messages: [{ id: uuidv4(), payload: "uuid" }] }; + assert.doesNotThrow(() => SEND_MESSAGES.serialize(t)); }); - it('throw on invalid string message id', () => { - const t = { ...t1, messages: [{ id: 'foo', payload: 'm' }] }; - assert.throws( - () => SEND_MESSAGES.serialize(t) - ); + it("throw on invalid string message id", () => { + const t = { ...t1, messages: [{ id: "foo", payload: "m" }] }; + assert.throws(() => SEND_MESSAGES.serialize(t)); }); - it('throw on invalid number message id', () => { - const t = { ...t1, messages: [{ id: -12, payload: 'n' }] }; - assert.throws( - () => SEND_MESSAGES.serialize(t) - ); + it("throw on invalid number message id", () => { + const t = { ...t1, messages: [{ id: -12, payload: "n" }] }; + assert.throws(() => SEND_MESSAGES.serialize(t)); }); - it('throw on invalid bigint message id', () => { - const t = { ...t1, messages: [{ id: -12n, payload: 'bn' }] }; - assert.throws( - () => SEND_MESSAGES.serialize(t) - ); + it("throw on invalid bigint message id", () => { + const t = { ...t1, messages: [{ id: -12n, payload: "bn" }] }; + assert.throws(() => SEND_MESSAGES.serialize(t)); }); - - it('serialize message with headers', () => { + it("serialize message with headers", () => { const t: SendMessages = { streamId: 911, topicId: 213, messages: [ - { payload: 'm', headers: { p: HeaderValue.Bool(true) } }, - { payload: 'q', headers: { 'v-aze': HeaderValue.Uint8(128) } }, - { payload: 'x', headers: { q: HeaderValue.Double(1/3) } }, - { payload: 's', headers: { x: HeaderValue.Uint32(123) } }, - { payload: 'r', headers: { y: HeaderValue.Uint64(42n) } }, - { payload: 'g', headers: { y: HeaderValue.Float(42.3) } }, - { payload: 'c', headers: { ID: HeaderValue.String(uuidv7()) } }, - { payload: 'l', headers: { val: HeaderValue.Raw(Buffer.from(uuidv4())) } } - ] + { + payload: "m", + headers: [ + { + key: HeaderKeyFactory.String("p"), + value: HeaderValue.Bool(true), + }, + ], + }, + { + payload: "q", + headers: [ + { + key: HeaderKeyFactory.String("v-aze"), + value: HeaderValue.Uint8(128), + }, + ], + }, + { + payload: "x", + headers: [ + { + key: HeaderKeyFactory.String("q"), + value: HeaderValue.Double(1 / 3), + }, + ], + }, + { + payload: "s", + headers: [ + { + key: HeaderKeyFactory.String("x"), + value: HeaderValue.Uint32(123), + }, + ], + }, + { + payload: "r", + headers: [ + { + key: HeaderKeyFactory.String("y"), + value: HeaderValue.Uint64(42n), + }, + ], + }, + { + payload: "g", + headers: [ + { + key: HeaderKeyFactory.String("y"), + value: HeaderValue.Float(42.3), + }, + ], + }, + { + payload: "c", + headers: [ + { + key: HeaderKeyFactory.String("ID"), + value: HeaderValue.String(uuidv7()), + }, + ], + }, + { + payload: "l", + headers: [ + { + key: HeaderKeyFactory.String("val"), + value: HeaderValue.Raw(Buffer.from(uuidv4())), + }, + ], + }, + ], }; - assert.doesNotThrow( - () => SEND_MESSAGES.serialize(t) - ); + assert.doesNotThrow(() => SEND_MESSAGES.serialize(t)); }); - - - }); }); diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml index 1b7a5c6dc3..eb8230fdcf 100644 --- a/foreign/python/Cargo.toml +++ b/foreign/python/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "apache-iggy" -version = "0.6.1-dev1" +version = "0.6.2-dev1" edition = "2021" authors = [ "Dario Lencina Talarico ", @@ -31,7 +31,7 @@ repository = "https://github.com/apache/iggy" [dependencies] bytes = "1.10.1" futures = "0.3.31" -iggy = { path = "../../core/sdk", version = "0.8.1-edge.1" } +iggy = { path = "../../core/sdk", version = "0.8.2-edge.1" } pyo3 = "0.26.0" pyo3-async-runtimes = { version = "0.26.0", features = [ "attributes", diff --git a/web/src/lib/components/Modals/InspectMessage.svelte b/web/src/lib/components/Modals/InspectMessage.svelte index 8c841d21e4..1e58c4f2be 100644 --- a/web/src/lib/components/Modals/InspectMessage.svelte +++ b/web/src/lib/components/Modals/InspectMessage.svelte @@ -20,9 +20,9 @@ @@ -75,7 +138,16 @@
Checksum -
{message?.checksum ?? 'N/A'}
+
+ {#if message?.checksum != null} + 0x{BigInt(message.checksum).toString(16).toUpperCase()} + + ({message.checksum}) + + {:else} + N/A + {/if} +
@@ -90,8 +162,38 @@
Headers -
- {formatHeaders(message?.user_headers)} +
+ {#if !message?.user_headers || message.user_headers.length === 0} + No headers + {:else} +
+ {#each message.user_headers as entry, index} + {@const keyValue = decodeHeaderValue(entry.key.kind, entry.key.value)} + {@const valueValue = decodeHeaderValue(entry.value.kind, entry.value.value)} + {@const isExpanded = expandedHeaders.has(index)} +
+
+ + {keyValue} + → + {valueValue} +
+ {#if isExpanded} +
+ key: {entry.key.kind}, value: {entry.value.kind} +
+ {/if} +
+ {/each} +
+ {/if}
diff --git a/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte b/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte index 77b98d633e..0f5a60c436 100644 --- a/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte +++ b/web/src/lib/components/RouteComponents/Settings/UsersTab.svelte @@ -74,13 +74,13 @@ const { checked } = e.target as HTMLInputElement; $selectedUsersId = checked - ? users.filter((user) => user.id !== 1).map((user) => `${user.id}`) + ? users.filter((user) => user.id !== 0).map((user) => `${user.id}`) : []; }; let allChecked = $derived( users - .filter((user) => user.id !== 1) + .filter((user) => user.id !== 0) .every((user) => $selectedUsersId.includes(user.id.toString())) ); @@ -127,13 +127,13 @@ for="{row.id}-{row.username}" class={twMerge( baseClass, - row.id === 1 && 'bg-shade-l800 dark:bg-shade-d1000 pointer-events-none', + row.id === 0 && 'bg-shade-l800 dark:bg-shade-d1000 pointer-events-none', $selectedUsersId.includes(row.id.toString()) && 'ring-2 ring-inset ring-green500 bg-green-300/30! ' )} >
- {#if row.id !== 1} + {#if row.id !== 0} ; + user_headers: HeaderEntry[]; payload: string; truncatedPayload: string; }; @@ -44,6 +44,11 @@ export type HeaderField = { value: string; }; +export type HeaderEntry = { + key: HeaderField; + value: HeaderField; +}; + export function messageMapper(item: any): Message { const payload = item.payload; const truncatedPayload = payload.length > 30 ? `${payload.slice(0, 30)} [...]` : payload; diff --git a/web/src/lib/domain/User.ts b/web/src/lib/domain/User.ts index db4a248df3..50e16fb48a 100644 --- a/web/src/lib/domain/User.ts +++ b/web/src/lib/domain/User.ts @@ -31,6 +31,6 @@ export function userMapper(item: any): User { id: item.id, createdAt: formatDate(item.created_at), status: item.status, - username: `${item.username} ${item.id === 1 ? '(root)' : ''}` + username: `${item.username} ${item.id === 0 ? '(root)' : ''}` }; } diff --git a/web/src/routes/dashboard/settings/users/+page.svelte b/web/src/routes/dashboard/settings/users/+page.svelte index 3090690d31..d57db86c20 100644 --- a/web/src/routes/dashboard/settings/users/+page.svelte +++ b/web/src/routes/dashboard/settings/users/+page.svelte @@ -76,13 +76,13 @@ const { checked } = e.target as HTMLInputElement; $selectedUsersId = checked - ? data.users.filter((user) => user.id !== 1).map((user) => `${user.id}`) + ? data.users.filter((user) => user.id !== 0).map((user) => `${user.id}`) : []; }; let allChecked = $derived( data.users - .filter((user) => user.id !== 1) + .filter((user) => user.id !== 0) .every((user) => $selectedUsersId.includes(user.id.toString())) ); @@ -173,13 +173,13 @@ for="{row.id}-{row.username}" class={twMerge( baseClass, - row.id === 1 && 'bg-shade-l800 dark:bg-shade-d1000 pointer-events-none', + row.id === 0 && 'bg-shade-l800 dark:bg-shade-d1000 pointer-events-none', $selectedUsersId.includes(row.id.toString()) && 'ring-2 ring-inset ring-green500 bg-green-300/30! ' )} >
- {#if row.id !== 1} + {#if row.id !== 0}
- {#if row.id !== 1} + {#if row.id !== 0} {#snippet trigger()}