diff --git a/async_postgres/pg_bytes.nim b/async_postgres/pg_bytes.nim new file mode 100644 index 0000000..0fa9868 --- /dev/null +++ b/async_postgres/pg_bytes.nim @@ -0,0 +1,48 @@ +## Low-level byte buffer helpers for big-endian encoding / bulk copy. +## +## This module is dependency-free (stdlib only) and intended to be imported +## from both ``pg_protocol`` and ``pg_types/*`` without introducing circular +## dependencies. Prefer these helpers over hand-written ``copyMem`` calls +## for readability and to keep ``addr`` use localized. + +template writeBE16*(buf: var openArray[byte], pos: int, v: int16) = + buf[pos] = byte((v shr 8) and 0xFF) + buf[pos + 1] = byte(v and 0xFF) + +template writeBE32*(buf: var openArray[byte], pos: int, v: int32) = + buf[pos] = byte((v shr 24) and 0xFF) + buf[pos + 1] = byte((v shr 16) and 0xFF) + buf[pos + 2] = byte((v shr 8) and 0xFF) + buf[pos + 3] = byte(v and 0xFF) + +template writeBE64*(buf: var openArray[byte], pos: int, v: int64) = + buf[pos] = byte((v shr 56) and 0xFF) + buf[pos + 1] = byte((v shr 48) and 0xFF) + buf[pos + 2] = byte((v shr 40) and 0xFF) + buf[pos + 3] = byte((v shr 32) and 0xFF) + buf[pos + 4] = byte((v shr 24) and 0xFF) + buf[pos + 5] = byte((v shr 16) and 0xFF) + buf[pos + 6] = byte((v shr 8) and 0xFF) + buf[pos + 7] = byte(v and 0xFF) + +template writeBytesAt*(dst: var openArray[byte], pos: int, src: openArray[byte]) = + ## Copy src bytes into dst starting at pos. No-op when src is empty. + if src.len > 0: + copyMem(addr dst[pos], addr src[0], src.len) + +template appendBytes*(buf: var seq[byte], src: openArray[byte]) = + ## Append src bytes to the end of buf. No-op when src is empty. + if src.len > 0: + buf.add(src) + +proc readString*(src: openArray[byte], off, len: int): string = + ## Copy `len` bytes from src starting at off into a new string. + result = newString(len) + if len > 0: + copyMem(addr result[0], addr src[off], len) + +proc readBytes*(src: openArray[byte], off, len: int): seq[byte] = + ## Copy `len` bytes from src starting at off into a new seq[byte]. + result = newSeq[byte](len) + if len > 0: + copyMem(addr result[0], addr src[off], len) diff --git a/async_postgres/pg_protocol.nim b/async_postgres/pg_protocol.nim index e70c3c9..dddfa75 100644 --- a/async_postgres/pg_protocol.nim +++ b/async_postgres/pg_protocol.nim @@ -1,5 +1,7 @@ import std/[options, tables] +import pg_bytes + type ProtocolError* = object of CatchableError ## Raised on PostgreSQL wire protocol violations. @@ -373,7 +375,7 @@ proc addCString*(buf: var seq[byte], s: string) = let oldLen = buf.len buf.setLen(oldLen + s.len + 1) if s.len > 0: - copyMem(addr buf[oldLen], addr s[0], s.len) + buf.writeBytesAt(oldLen, s.toOpenArrayByte(0, s.high)) buf[oldLen + s.len] = 0'u8 proc decodeInt16*(buf: openArray[byte], offset: int): int16 = @@ -404,9 +406,7 @@ proc decodeCString*(buf: openArray[byte], offset: int): (string, int) = if i >= buf.len: raise newException(ProtocolError, "decodeCString: missing null terminator") let slen = i - offset - var s = newString(slen) - if slen > 0: - copyMem(addr s[0], addr buf[offset], slen) + let s = readString(buf, offset, slen) inc i # skip null terminator result = (s, i - offset) @@ -477,7 +477,7 @@ proc encodeQuery*(sql: string): seq[byte] = proc addFixedMsg(buf: var seq[byte], msg: array[5, byte]) {.inline.} = let oldLen = buf.len buf.setLen(oldLen + 5) - copyMem(addr buf[oldLen], addr msg[0], 5) + buf.writeBytesAt(oldLen, msg) proc addParse*( buf: var seq[byte], @@ -522,10 +522,7 @@ proc addBind*( else: let data = v.get buf.addInt32(int32(data.len)) - if data.len > 0: - let oldLen = buf.len - buf.setLen(oldLen + data.len) - copyMem(addr buf[oldLen], addr data[0], data.len) + buf.appendBytes(data) # Result format codes buf.addInt16(int16(resultFormats.len)) for f in resultFormats: @@ -578,7 +575,7 @@ proc addBindRaw*( ) let oldLen = buf.len buf.setLen(oldLen + r.len) - copyMem(addr buf[oldLen], addr paramData[r.off], r.len) + buf.writeBytesAt(oldLen, paramData.toOpenArray(r.off, r.off + r.len - 1)) buf.addInt16(int16(resultFormats.len)) for f in resultFormats: buf.addInt16(f) @@ -684,8 +681,7 @@ proc encodeCopyData*(buf: var seq[byte], data: openArray[byte]) = buf[oldLen + 2] = byte((msgLen shr 16) and 0xFF) buf[oldLen + 3] = byte((msgLen shr 8) and 0xFF) buf[oldLen + 4] = byte(msgLen and 0xFF) - if data.len > 0: - copyMem(addr buf[oldLen + 5], addr data[0], data.len) + buf.writeBytesAt(oldLen + 5, data) proc encodeCopyDone*(): seq[byte] = ## Encode a standalone CopyDone message. @@ -973,7 +969,7 @@ proc clone*(row: Row): Row = rd.cellIndex[i * 2] = 0'i32 rd.cellIndex[i * 2 + 1] = 0'i32 else: - copyMem(addr rd.buf[pos], addr src.buf[srcOff], int(clen)) + rd.buf.writeBytesAt(pos, src.buf.toOpenArray(srcOff, srcOff + int(clen) - 1)) rd.cellIndex[i * 2] = int32(pos) rd.cellIndex[i * 2 + 1] = clen pos += int(clen) @@ -1016,7 +1012,7 @@ proc parseDataRowInto*(body: openArray[byte], rd: RowData) = let dataLen = body.len - 2 rd.buf.setLen(bufBase + dataLen) if dataLen > 0: - copyMem(addr rd.buf[bufBase], addr body[2], dataLen) + rd.buf.writeBytesAt(bufBase, body.toOpenArray(2, 2 + dataLen - 1)) # Walk the copied buffer to build cellIndex var pos = bufBase # current position in rd.buf let bufEnd = bufBase + dataLen @@ -1164,9 +1160,7 @@ proc formatError*(fields: seq[ErrorField]): string = proc addCopyBinaryHeader*(buf: var seq[byte]) = ## Append the PostgreSQL binary COPY header (signature + flags + extension area). - let oldLen = buf.len - buf.setLen(oldLen + pgCopyBinaryHeader.len) - copyMem(addr buf[oldLen], addr pgCopyBinaryHeader[0], pgCopyBinaryHeader.len) + buf.appendBytes(pgCopyBinaryHeader) proc addCopyBinaryTrailer*(buf: var seq[byte]) = ## Append the binary COPY trailer (int16 = -1). @@ -1214,18 +1208,13 @@ proc addCopyFieldBool*(buf: var seq[byte], val: bool) = proc addCopyFieldText*(buf: var seq[byte], val: openArray[byte]) = ## Append a raw byte field in binary COPY format. buf.addInt32(int32(val.len)) - if val.len > 0: - let oldLen = buf.len - buf.setLen(oldLen + val.len) - copyMem(addr buf[oldLen], addr val[0], val.len) + buf.appendBytes(val) proc addCopyFieldString*(buf: var seq[byte], val: string) = ## Append a string field in binary COPY format. buf.addInt32(int32(val.len)) if val.len > 0: - let oldLen = buf.len - buf.setLen(oldLen + val.len) - copyMem(addr buf[oldLen], addr val[0], val.len) + buf.appendBytes(val.toOpenArrayByte(0, val.high)) # Replication protocol helpers diff --git a/async_postgres/pg_types/decoding.nim b/async_postgres/pg_types/decoding.nim index 93f8d30..ec067fd 100644 --- a/async_postgres/pg_types/decoding.nim +++ b/async_postgres/pg_types/decoding.nim @@ -1,18 +1,9 @@ import std/[options, strutils, tables, times, net] +import ../pg_bytes import ./core -proc readString*(src: openArray[byte], off, len: int): string = - ## Copy `len` bytes from src starting at off into a new string. - result = newString(len) - if len > 0: - copyMem(addr result[0], addr src[off], len) - -proc readBytes*(src: openArray[byte], off, len: int): seq[byte] = - ## Copy `len` bytes from src starting at off into a new seq[byte]. - result = newSeq[byte](len) - if len > 0: - copyMem(addr result[0], addr src[off], len) +export pg_bytes proc decodeHstoreBinary*(data: openArray[byte]): PgHstore = ## Decode PostgreSQL binary hstore format. diff --git a/async_postgres/pg_types/encoding.nim b/async_postgres/pg_types/encoding.nim index b156d7b..fd99de7 100644 --- a/async_postgres/pg_types/encoding.nim +++ b/async_postgres/pg_types/encoding.nim @@ -1,37 +1,9 @@ import std/[options, json, macros, strutils, tables, times, net] -import ../pg_protocol +import ../[pg_bytes, pg_protocol] import ./core -template writeBE16*(buf: var openArray[byte], pos: int, v: int16) = - buf[pos] = byte((v shr 8) and 0xFF) - buf[pos + 1] = byte(v and 0xFF) - -template writeBE32*(buf: var openArray[byte], pos: int, v: int32) = - buf[pos] = byte((v shr 24) and 0xFF) - buf[pos + 1] = byte((v shr 16) and 0xFF) - buf[pos + 2] = byte((v shr 8) and 0xFF) - buf[pos + 3] = byte(v and 0xFF) - -template writeBE64*(buf: var openArray[byte], pos: int, v: int64) = - buf[pos] = byte((v shr 56) and 0xFF) - buf[pos + 1] = byte((v shr 48) and 0xFF) - buf[pos + 2] = byte((v shr 40) and 0xFF) - buf[pos + 3] = byte((v shr 32) and 0xFF) - buf[pos + 4] = byte((v shr 24) and 0xFF) - buf[pos + 5] = byte((v shr 16) and 0xFF) - buf[pos + 6] = byte((v shr 8) and 0xFF) - buf[pos + 7] = byte(v and 0xFF) - -template writeBytesAt*(dst: var openArray[byte], pos: int, src: openArray[byte]) = - ## Copy src bytes into dst starting at pos. No-op when src is empty. - if src.len > 0: - copyMem(addr dst[pos], addr src[0], src.len) - -template appendBytes*(buf: var seq[byte], src: openArray[byte]) = - ## Append src bytes to the end of buf. No-op when src is empty. - if src.len > 0: - buf.add(src) +export pg_bytes proc toPgParamInline*(v: int16): PgParamInline = result.oid = OidInt2 @@ -79,10 +51,10 @@ proc toPgParamInline*(v: string): PgParamInline = if v.len == 0: discard elif v.len <= PgInlineBufSize: - copyMem(addr result.inlineBuf[0], addr v[0], v.len) + result.inlineBuf.writeBytesAt(0, v.toOpenArrayByte(0, v.high)) else: result.overflow = newSeq[byte](v.len) - copyMem(addr result.overflow[0], addr v[0], v.len) + result.overflow.writeBytesAt(0, v.toOpenArrayByte(0, v.high)) proc toPgParamInline*(v: seq[byte]): PgParamInline = result.oid = OidBytea @@ -91,7 +63,7 @@ proc toPgParamInline*(v: seq[byte]): PgParamInline = if v.len == 0: discard elif v.len <= PgInlineBufSize: - copyMem(addr result.inlineBuf[0], addr v[0], v.len) + result.inlineBuf.writeBytesAt(0, v) else: result.overflow = v @@ -105,10 +77,10 @@ proc toPgParamInline*(v: PgUuid): PgParamInline = if s.len == 0: discard elif s.len <= PgInlineBufSize: - copyMem(addr result.inlineBuf[0], addr s[0], s.len) + result.inlineBuf.writeBytesAt(0, s.toOpenArrayByte(0, s.high)) else: result.overflow = newSeq[byte](s.len) - copyMem(addr result.overflow[0], addr s[0], s.len) + result.overflow.writeBytesAt(0, s.toOpenArrayByte(0, s.high)) proc toPgParamInline*(v: PgMoney): PgParamInline = result.oid = OidMoney