Merge branch 'master' into midi

Author: twystd

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 
 env:
-  GOLANGCILINT_VERSION: "1.60.1"
+  GOLANGCILINT_VERSION: "1.60.3"
 
 jobs:
   lint:

.golangci.yml

@@ -37,6 +37,9 @@ linters-settings:
         excludes:
             - G401
             - G501
+            - G115
+            - G406
+            - G506
     misspell:
         ignore-words:
             # elf RELA

Makefile

@@ -61,7 +61,7 @@ gogenerate: always
 lint: always
 # bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
 # bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
-	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.1 run
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3 run
 
 depgraph.svg: always
 	go run github.com/kisielk/godepgraph@latest github.com/wader/fq | dot -Tsvg -o godepgraph.svg

doc/formats.md

@@ -1261,6 +1261,15 @@ You will often find this format embedded inside the TZX tape format.
 
 The default file extension is `.tap`.
 
+### Processing JSON files
+
+When needing to process a generated JSON file it's recommended to convert the
+plain data bytes to an array by setting `bits_format=byte_array`:
+
+```bash
+fq -o bits_format=byte_array -d tap -V d /path/to/file.tap
+```
+
 ### Authors
 
 - Michael R. Cook work.mrc@pm.me, original author
@@ -1451,6 +1460,15 @@ the company Ramsoft became the maintainers, and created revision `v1.20`.
 
 The default file extension is `.tzx`.
 
+### Processing JSON files
+
+When needing to process a generated JSON file it's recommended to convert the
+plain data bytes to an array by setting `bits_format=byte_array`:
+
+```bash
+fq -o bits_format=byte_array -d tzx -V d /path/to/file.tzx
+```
+
 ### Authors
 
 - Michael R. Cook work.mrc@pm.me, original author

doc/usage.md

@@ -55,6 +55,8 @@ fq -Vr .path.to.string file
 fq -o bits_format=truncate tovalue file
 # JSON but raw bit fields as md5 hex string
 fq -o bits_format=md5 tovalue file
+# JSON but raw bit fields as byte arrays
+fq -o bits_format=byte_array tovalue file
 # look up a path
 fq '.some[1].path' file
 # look up a path and output JSON
@@ -403,8 +405,8 @@ There is also `tobitsrange` and `tobytesrange` which does the same thing but wil
   - `[0x12, 0x34, 0x56] | tobits[4:12]` will be a binary with the byte `0x23`
   - `[0x12, 0x34, 0x56] | tobits[4:20]` will be a binary with the byte `0x23`, `0x45`
   - `[0x12, 0x34, 0x56] | tobits[4:20] | tobytes[1:]` will be a binary with the byte `0x45`,
-
-Both `.[index]` and `.[start:end]` support negative indices to index from end.
+  - Both `.[index]` and `.[start:end]` support negative indices to index from end.
+- `explode` output an array with all byte or bits as integers.
 
 #### Binary array
 
@@ -898,12 +900,13 @@ fq has some general options in addition to decode and decoders specific options.
 
 How to represent raw binary as JSON.
 
-- `-o bits_format=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
-- `-o bits_format=md5` MD5 hex string (zero bit padded).
-- `-o bits_format=hex` Hex string.
 - `-o bits_format=base64` Base64 string.
-- `-p bits_format=truncate` Truncated string.
+- `-o bits_format=byte_array` Array of bytes (zero bit padded if size is not byte aligned).
+- `-o bits_format=hex` Hex string.
+- `-o bits_format=md5` MD5 hex string (zero bit padded).
 - `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.
+- `-o bits_format=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn into JSON (default).
+- `-p bits_format=truncate` Truncated string.
 
 ```sh
 $ fq -V -o bits_format=base64 . file

format/json/jq.jq

@@ -81,7 +81,7 @@ def from_jq:
         ( $v.term.object.key_vals // []
         | map(
             { key: (.key // .key_string.str)
-            , value: (.val.queries[0] | _f)
+            , value: (.val | _f)
             }
           )
         | from_entries

format/tap/tap.go

@@ -6,6 +6,7 @@ import (
 	"bufio"
 	"bytes"
 	"embed"
+	"fmt"
 
 	"golang.org/x/text/encoding/charmap"
 
@@ -54,9 +55,11 @@ func decodeTapBlock(d *decode.D) {
 	// read header, fragment, or data block
 	switch length {
 	case 0:
-		// fragment with no data
+		d.Fatalf("TAP fragments with 0 bytes are not supported")
 	case 1:
-		d.FieldRawLen("data", 8)
+		d.FieldStruct("data", func(d *decode.D) {
+			d.FieldRawLen("bytes", 8)
+		})
 	case 19:
 		d.FieldStruct("header", func(d *decode.D) {
 			decodeHeader(d)
@@ -72,15 +75,34 @@ func decodeTapBlock(d *decode.D) {
 func decodeHeader(d *decode.D) {
 	blockStartPosition := d.Pos()
 
-	// Always 0: byte indicating a standard ROM loading header
-	d.FieldU8("flag", scalar.UintMapSymStr{0: "standard_speed_data"})
+	// flag indicating the type of header block, usually 0 (standard speed data)
+	d.FieldU8("flag", scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
+		if s.Actual == 0x00 {
+			s.Sym = "standard_speed_data"
+		} else {
+			s.Sym = "custom_data_block"
+		}
+		return s, nil
+	}))
+
 	// Header type
-	dataType := d.FieldU8("data_type", scalar.UintMapSymStr{
-		0x00: "program",
-		0x01: "numeric",
-		0x02: "alphanumeric",
-		0x03: "data",
-	})
+	dataType := d.FieldU8("data_type", scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
+		switch s.Actual {
+		case 0x00:
+			s.Sym = "program"
+		case 0x01:
+			s.Sym = "numeric"
+		case 0x02:
+			s.Sym = "alphanumeric"
+		case 0x03:
+			s.Sym = "data"
+		default:
+			// unofficial header types
+			s.Sym = fmt.Sprintf("unknown%02X", s.Actual)
+		}
+		return s, nil
+	}))
+
 	// Loading name of the program. Filled with spaces (0x20) to 10 characters.
 	d.FieldStr("program_name", 10, charmap.ISO8859_1)
 
@@ -120,7 +142,10 @@ func decodeHeader(d *decode.D) {
 		//	UnusedWord: 32768.
 		d.FieldU16("unused")
 	default:
-		d.Fatalf("invalid TAP header type, got: %d", dataType)
+		// Unofficial header types
+		d.FieldU16("data_length")
+		d.FieldU16("unknown1", scalar.UintHex)
+		d.FieldU16("unknown2", scalar.UintHex)
 	}
 
 	// Simply all bytes XORed (including flag byte).
@@ -140,7 +165,8 @@ func decodeDataBlock(d *decode.D, length uint64) {
 		return s, nil
 	}))
 	// The essential data: length minus the flag/checksum bytes (may be empty)
-	d.FieldRawLen("data", int64(length-2)*8)
+	d.FieldRawLen("bytes", int64(length-2)*8)
+
 	// Simply all bytes (including flag byte) XORed
 	d.FieldU8("checksum", d.UintValidate(calculateChecksum(d, blockStartPosition, d.Pos()-blockStartPosition)), scalar.UintHex)
 }

format/tap/tap.md

@@ -7,6 +7,15 @@ You will often find this format embedded inside the TZX tape format.
 
 The default file extension is `.tap`.
 
+### Processing JSON files
+
+When needing to process a generated JSON file it's recommended to convert the
+plain data bytes to an array by setting `bits_format=byte_array`:
+
+```bash
+fq -o bits_format=byte_array -d tap -V d /path/to/file.tap
+```
+
 ### Authors
 
 - Michael R. Cook work.mrc@pm.me, original author

format/tap/testdata/basic_prog1.fqtest

@@ -15,7 +15,7 @@ $ fq -d tap dv basic_prog1.tap
 0x10|               28 00                           |     (.         |      length: 40 0x15-0x17 (2)
     |                                               |                |      data{}: 0x17-0x3f (40)
 0x10|                     ff                        |       .        |        flag: "standard_speed_data" (255) 0x17-0x18 (1)
-0x10|                        00 0a 14 00 20 f5 22 66|        .... ."f|        data: raw bits 0x18-0x3e (38)
+0x10|                        00 0a 14 00 20 f5 22 66|        .... ."f|        bytes: raw bits 0x18-0x3e (38)
 0x20|71 20 69 73 20 74 68 65 20 62 65 73 74 21 22 0d|q is the best!".|
 0x30|00 14 0a 00 ec 31 30 0e 00 00 0a 00 00 0d      |.....10.......  |
 0x30|                                          b6|  |              .||        checksum: 0xb6 (valid) 0x3e-0x3f (1)