Skip to content

Commit bea6643

Browse files
schmikeijsirianni
schmikei
and
jsirianni
authored
feat: Allow HTTP receiver to specify text payloads (#2931)
* spike out http receiver supporting text payloads * newlines at EOF of test file inputs * update golden with newline removal * update to check header before trying to parse as JSON * accomodate windows different line termination characters in test case * for now just check for non-empty content type before treating as JSON * use switch statement for content type and persist old behavior * update readme based off recent changes * chore(test): Split tests in ci (#2933) * add raw config parameter * normalize lines for windows in golden test * remove test line expectation * pr feedback; update comments and wording * update content-type to always try to parse JSON even if not application/json --------- Co-authored-by: Joseph Sirianni <joe.sirianni@observiq.com>
1 parent 3864e1c commit bea6643

15 files changed

Lines changed: 674 additions & 41 deletions

File tree

‎.github/workflows/tests.yml‎

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,41 @@ jobs:
77
if: github.actor != 'dependabot[bot]'
88
strategy:
99
fail-fast: false
10+
max-parallel: 4
1011
matrix:
1112
include:
13+
# Ubuntu - All tests
1214
- os: ubuntu
1315
runner: namespace-profile-bindplane-default-ubuntu-24-04-32x64
14-
go_mod_path: /home/runner/go/pkg/mod
15-
go_bin_path: /home/runner/go/bin
1616
test_cmd: sudo make test-updater-integration
17+
# macOS - All tests
1718
- os: macos
1819
runner: namespace-profile-bindplane-default-macos-15
19-
go_mod_path: /Users/runner/go/pkg/mod
20-
go_bin_path: /Users/runner/go/bin
2120
test_cmd: make test-updater-integration
21+
# Windows - Receivers
2222
- os: windows
23-
runner: namespace-profile-bindplane-default-windows-2022-32x64
24-
go_mod_path: C:\Users\runneradmin\go\pkg\mod
25-
go_bin_path: C:\Users\runneradmin\go\bin
26-
go_build_path: C:\Users\runneradmin\AppData\Local\go-build
23+
target: receivers
24+
runner: namespace-profile-bindplane-default-windows-2022-8x16
25+
test_cmd: make test-updater-integration
26+
# Windows - Processors
27+
- os: windows
28+
target: processors
29+
runner: namespace-profile-bindplane-default-windows-2022-8x16
30+
test_cmd: make test-updater-integration
31+
# Windows - Exporters
32+
- os: windows
33+
target: exporters
34+
runner: namespace-profile-bindplane-default-windows-2022-8x16
35+
test_cmd: make test-updater-integration
36+
# Windows - Extensions
37+
- os: windows
38+
target: extensions
39+
runner: namespace-profile-bindplane-default-windows-2022-8x16
40+
test_cmd: make test-updater-integration
41+
# Windows - Other
42+
- os: windows
43+
target: other
44+
runner: namespace-profile-bindplane-default-windows-2022-8x16
2745
test_cmd: make test-updater-integration
2846
runs-on: ${{ matrix.runner }}
2947
steps:
@@ -51,19 +69,19 @@ jobs:
5169
if: matrix.os == 'ubuntu'
5270
uses: namespacelabs/nscloud-cache-action@v1
5371
with:
54-
path: ${{ matrix.go_mod_path }}
72+
path: /home/runner/go/pkg/mod
5573

5674
- name: Cache Go modules (macOS)
5775
if: matrix.os == 'macos'
5876
uses: namespacelabs/nscloud-cache-action@v1
5977
with:
60-
path: ${{ matrix.go_mod_path }}
78+
path: /Users/runner/go/pkg/mod
6179

6280
- name: Cache Go modules (Windows)
6381
if: matrix.os == 'windows'
6482
uses: actions/cache@v4
6583
with:
66-
path: ${{ matrix.go_mod_path }}
84+
path: C:\Users\runneradmin\go\pkg\mod
6785
key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
6886
restore-keys: |
6987
${{ runner.os }}-go-mod-
@@ -72,7 +90,7 @@ jobs:
7290
if: matrix.os == 'windows'
7391
uses: actions/cache@v4
7492
with:
75-
path: ${{ matrix.go_build_path }}
93+
path: C:\Users\runneradmin\AppData\Local\go-build
7694
key: ${{ runner.os }}-go-build-${{ hashFiles('**/*.go') }}
7795
restore-keys: |
7896
${{ runner.os }}-go-build-
@@ -81,30 +99,38 @@ jobs:
8199
if: matrix.os == 'ubuntu'
82100
uses: namespacelabs/nscloud-cache-action@v1
83101
with:
84-
path: ${{ matrix.go_bin_path }}
102+
path: /home/runner/go/bin
85103

86104
- name: Cache Tools (macOS)
87105
if: matrix.os == 'macos'
88106
uses: namespacelabs/nscloud-cache-action@v1
89107
with:
90-
path: ${{ matrix.go_bin_path }}
108+
path: /Users/runner/go/bin
91109

92110
- name: Cache Tools (Windows)
93111
if: matrix.os == 'windows'
94112
uses: actions/cache@v4
95113
with:
96-
path: ${{ matrix.go_bin_path }}
114+
path: C:\Users\runneradmin\go\bin
97115
key: ${{ runner.os }}-go-tools-${{ hashFiles('internal/tools/go.sum') }}
98116
restore-keys: |
99117
${{ runner.os }}-go-tools-
100118
101119
- name: Install Tools
102120
run: make install-tools-ci
103121

104-
- name: Run Tests
122+
- name: Run Component Tests
123+
if: matrix.os == 'windows'
124+
run: make test-${{ matrix.target }}
125+
126+
- name: Run All Tests
127+
if: matrix.os != 'windows'
105128
run: make test
106129

130+
# Run updater integration tests for Ubuntu and macOS (always)
131+
# For Windows, only run when target is 'other'
107132
- name: Run Updater Integration Tests
133+
if: matrix.os != 'windows' || matrix.target == 'other'
108134
run: ${{ matrix.test_cmd }}
109135

110136
test-summary:

‎Makefile‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,64 @@ misspell-fix:
138138
test:
139139
$(MAKE) for-all CMD="gotestsum --rerun-fails --packages="./..." -- -race"
140140

141+
.PHONY: test-receivers
142+
test-receivers:
143+
@set -e; for dir in $(ALL_MODULES); do \
144+
if echo "$${dir}" | grep -qE "^\.?/?receiver/"; then \
145+
(cd "$${dir}" && \
146+
echo "running tests in $${dir}" && \
147+
gotestsum --rerun-fails --packages="./..." -- -race) || exit 1; \
148+
fi; \
149+
done
150+
151+
.PHONY: test-processors
152+
test-processors:
153+
@set -e; for dir in $(ALL_MODULES); do \
154+
if echo "$${dir}" | grep -qE "^\.?/?processor/"; then \
155+
(cd "$${dir}" && \
156+
echo "running tests in $${dir}" && \
157+
gotestsum --rerun-fails --packages="./..." -- -race) || exit 1; \
158+
fi; \
159+
done
160+
161+
.PHONY: test-exporters
162+
test-exporters:
163+
@set -e; for dir in $(ALL_MODULES); do \
164+
if echo "$${dir}" | grep -qE "^\.?/?exporter/"; then \
165+
(cd "$${dir}" && \
166+
echo "running tests in $${dir}" && \
167+
gotestsum --rerun-fails --packages="./..." -- -race) || exit 1; \
168+
fi; \
169+
done
170+
171+
.PHONY: test-extensions
172+
test-extensions:
173+
@set -e; for dir in $(ALL_MODULES); do \
174+
if echo "$${dir}" | grep -qE "^\.?/?extension/"; then \
175+
(cd "$${dir}" && \
176+
echo "running tests in $${dir}" && \
177+
gotestsum --rerun-fails --packages="./..." -- -race) || exit 1; \
178+
fi; \
179+
done
180+
181+
.PHONY: test-other
182+
test-other:
183+
@PACKAGES=$$(go list ./... | grep -v "/receiver" | grep -v "/processor" | grep -v "/exporter" | grep -v "/extension" | tr '\n' ' '); \
184+
if [ -n "$$PACKAGES" ]; then \
185+
gotestsum --rerun-fails --packages="$$PACKAGES" -- -race; \
186+
fi
187+
@set -e; for dir in $(ALL_MODULES); do \
188+
if [ "$${dir}" = "." ]; then \
189+
continue; \
190+
elif echo "$${dir}" | grep -qE "^(receiver|processor|exporter|extension)/"; then \
191+
continue; \
192+
else \
193+
(cd "$${dir}" && \
194+
echo "running tests in $${dir}" && \
195+
gotestsum --rerun-fails --packages="./..." -- -race) || exit 1; \
196+
fi; \
197+
done
198+
141199
.PHONY: test-no-race
142200
test-no-race:
143201
$(MAKE) for-all CMD="gotestsum --rerun-fails --packages="./..." "

‎receiver/httpreceiver/README.md‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# HTTP Receiver
2-
This receiver is capable of collecting logs for a variety of services, serving as a default HTTP log receiver. Anything that is able to send JSON structured logs to an endpoint using HTTP will be able to utilize this receiver.
2+
This receiver is capable of collecting logs for a variety of services, serving as a default HTTP log receiver. Anything that is able to send logs to an endpoint using HTTP will be able to utilize this receiver.
33

44
## Minimum Agent Versions
55
- Introduced: [v1.39.0](https://github.com/observIQ/bindplane-otel-collector/releases/tag/v1.39.0)
@@ -13,13 +13,45 @@ This receiver is capable of collecting logs for a variety of services, serving a
1313

1414
## Prerequisites
1515
- The log source can be configured to send logs to an endpoint using HTTP
16-
- The logs sent by the log source are JSON structured
16+
17+
## Supported Formats
18+
19+
The receiver handles different payload formats based on the `Content-Type` header:
20+
21+
### JSON Payloads (`Content-Type: application/json`)
22+
- **JSON Object**: Single log entry as a JSON object
23+
```json
24+
{"message": "error occurred", "level": "error", "timestamp": 1699276151086}
25+
```
26+
- **JSON Array**: Multiple log entries as an array of JSON objects
27+
```json
28+
[
29+
{"message": "first log", "level": "info"},
30+
{"message": "second log", "level": "debug"}
31+
]
32+
```
33+
34+
### Text Payloads (`Content-Type: text/*`)
35+
Plain text payloads with a `text/*` content type (e.g., `text/plain`) are automatically wrapped in a log structure with a `body` field:
36+
```
37+
This is a plain text log message
38+
```
39+
40+
### No Content-Type Header
41+
For backward compatibility, if no `Content-Type` header is provided, the receiver will attempt to parse the payload as JSON. If the payload is not valid JSON, the request will be rejected with a 422 status code.
42+
43+
### Important Notes
44+
- If `Content-Type: application/json` is specified but the payload is not valid JSON, the request will be rejected with a 422 status code.
45+
- Content types with `+json` suffix (e.g., `application/ld+json`) are treated as JSON.
46+
- Any other content types not explicitly supported will be rejected with a 422 status code.
47+
- When the `raw` configuration parameter is set to `true`, all content-type detection is bypassed and payloads are always treated as plain text.
1748

1849
## Configuration
1950
| Field | Type | Default | Required | Description |
2051
|----------------------|-----------|------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
2152
| endpoint | string | | `true` | The hostname and port the receiver should listen on for logs being sent as HTTP POST requests. |
2253
| path | string | | `false` | Specifies a path the receiver should be listening to for logs. Useful when the log source also sends other data to the endpoint, such as metrics. |
54+
| raw | bool | `false` | `false` | When set to `true`, all payloads will be treated as plain text regardless of the `Content-Type` header. The entire payload will be stored as a string in the log body. |
2355
| tls.key_file | string | | `false` | Configure the receiver to use TLS. |
2456
| tls.cert_file | string | | `false` | Configure the receiver to use TLS. |
2557

@@ -59,3 +91,23 @@ service:
5991
receivers: [http]
6092
exporters: [googlecloud]
6193
```
94+
95+
### Example Configuration With Raw Mode
96+
Use `raw: true` to force all incoming payloads to be treated as plain text, regardless of `Content-Type` header. This is useful when you want to preserve the exact format of incoming logs without any JSON parsing.
97+
98+
```yaml
99+
receivers:
100+
http:
101+
endpoint: "localhost:12345"
102+
path: "/logs"
103+
raw: true
104+
exporters:
105+
googlecloud:
106+
project: my-gcp-project
107+
108+
service:
109+
pipelines:
110+
logs:
111+
receivers: [http]
112+
exporters: [googlecloud]
113+
```

‎receiver/httpreceiver/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
// Config defines the configuration for an HTTP receiver
2727
type Config struct {
2828
Path string `mapstructure:"path"`
29+
Raw bool `mapstructure:"raw"`
2930
ServerConfig confighttp.ServerConfig `mapstructure:",squash"`
3031
}
3132

‎receiver/httpreceiver/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/observiq/bindplane-otel-collector/receiver/httpreceiver
33
go 1.24.4
44

55
require (
6+
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.140.1
67
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.140.1
78
github.com/stretchr/testify v1.11.1
89
go.opentelemetry.io/collector/component v1.46.0

0 commit comments

Comments
 (0)