diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 826ff51f929..01ac5e31948 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -33,6 +33,7 @@ specifiers: '@rush-temp/openapi3': file:./projects/openapi3.tgz '@rush-temp/playground': file:./projects/playground.tgz '@rush-temp/prettier-plugin-typespec': file:./projects/prettier-plugin-typespec.tgz + '@rush-temp/protobuf': file:./projects/protobuf.tgz '@rush-temp/ref-doc': file:./projects/ref-doc.tgz '@rush-temp/rest': file:./projects/rest.tgz '@rush-temp/samples': file:./projects/samples.tgz @@ -47,6 +48,7 @@ specifiers: '@types/babel__code-frame': ~7.0.3 '@types/debounce': ~1.2.1 '@types/js-yaml': ~4.0.1 + '@types/micromatch': ^4.0.2 '@types/mocha': ~10.0.0 '@types/mustache': ~4.2.1 '@types/node': ~18.11.9 @@ -92,6 +94,7 @@ specifiers: lzutf8: 0.6.3 mdx-mermaid: 1.3.2 mermaid: ~9.1.6 + micromatch: ^4.0.5 mkdirp: ~2.1.6 mocha: ~10.2.0 mocha-junit-reporter: ~2.2.0 @@ -167,6 +170,7 @@ dependencies: '@rush-temp/openapi3': file:projects/openapi3.tgz '@rush-temp/playground': file:projects/playground.tgz_rollup@3.4.0 '@rush-temp/prettier-plugin-typespec': file:projects/prettier-plugin-typespec.tgz + '@rush-temp/protobuf': file:projects/protobuf.tgz '@rush-temp/ref-doc': file:projects/ref-doc.tgz '@rush-temp/rest': file:projects/rest.tgz '@rush-temp/samples': file:projects/samples.tgz @@ -181,6 +185,7 @@ dependencies: '@types/babel__code-frame': 7.0.3 '@types/debounce': 1.2.1 '@types/js-yaml': 4.0.5 + '@types/micromatch': 4.0.2 '@types/mocha': 10.0.1 '@types/mustache': 4.2.2 '@types/node': 18.11.19 @@ -226,6 +231,7 @@ dependencies: lzutf8: 0.6.3 mdx-mermaid: 1.3.2_mermaid@9.1.7+react@18.2.0 mermaid: 9.1.7 + micromatch: 4.0.5 mkdirp: 2.1.6 mocha: 10.2.0 mocha-junit-reporter: 2.2.0_mocha@10.2.0 @@ -5735,6 +5741,10 @@ packages: '@types/node': 18.11.19 dev: false + /@types/braces/3.0.1: + resolution: {integrity: sha512-+euflG6ygo4bn0JHtn4pYqcXwRtLvElQ7/nnjDu7iYG56H0+OhCd7d6Ug0IE3WcFpZozBKW2+80FUbv5QGk5AQ==} + dev: false + /@types/connect-history-api-fallback/1.3.5: resolution: {integrity: sha512-h8QJa8xSb1WD4fpKBDcATDNGXghFj6/3GRWG6dhmRcu0RX1Ubasur2Uvx5aeEwlf0MwblEC2bMzzMQntxnw/Cw==} dependencies: @@ -5852,6 +5862,12 @@ packages: '@types/unist': 2.0.6 dev: false + /@types/micromatch/4.0.2: + resolution: {integrity: sha512-oqXqVb0ci19GtH0vOA/U2TmHTcRY9kuZl4mqUxe0QmJAlIW13kzhuK5pi1i9+ngav8FjpSb9FVS/GE00GLX1VA==} + dependencies: + '@types/braces': 3.0.1 + dev: false + /@types/mime/3.0.1: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} dev: false @@ -16080,7 +16096,7 @@ packages: dev: false file:projects/openapi.tgz: - resolution: {integrity: sha512-XZUiXIfblSgzVpnvOzyrNujvfzxAs65FyCiKJZaulq/gvPWymRrhFAw9nMPTxGY9BusiISi2VIXnwM4GRlJXDw==, tarball: file:projects/openapi.tgz} + resolution: {integrity: sha512-dsDJIm7MZA6lFZvYLXu2so/Dg8QpnGgrd1xcWxi+rrQ/x+RPeiKQL9UtxM81ftVBI7SRi6HVqTwuo8AAY+R6ng==, tarball: file:projects/openapi.tgz} name: '@rush-temp/openapi' version: 0.0.0 dependencies: @@ -16098,7 +16114,7 @@ packages: dev: false file:projects/openapi3.tgz: - resolution: {integrity: sha512-wQijfiCS64ra9snbWzBRvntxYi1VqfCg4rYP7k41ipbtVRlXzBges2/JbmxQ4Lu7aa7lssx/TvnpS+8HMYTOog==, tarball: file:projects/openapi3.tgz} + resolution: {integrity: sha512-nd7sene3BOLtOAEEIm8cr222vc3w05gn2T4ji+psU3XzltBtBSGtH4HBaiqF06hVAYH5leDdQVeBNvP8sFfOBQ==, tarball: file:projects/openapi3.tgz} name: '@rush-temp/openapi3' version: 0.0.0 dependencies: @@ -16118,7 +16134,7 @@ packages: dev: false file:projects/playground.tgz_rollup@3.4.0: - resolution: {integrity: sha512-0HxFAq27WLaaYUpvHU9hOYdLU8Vuzf6RsurXS/6Nan2SlNTLpQGxoEiCcTwiZ/7NA7ipM02eOy5DzDTzHvZXCg==, tarball: file:projects/playground.tgz} + resolution: {integrity: sha512-ce4xC/dS51QetqunqwR9qPHo/KCndoEjznty+z0dZPtc7RRVqCj8vmDdYL1JYFiP1iD6JutD+uEFXZ7WK7X83w==, tarball: file:projects/playground.tgz} id: file:projects/playground.tgz name: '@rush-temp/playground' version: 0.0.0 @@ -16191,6 +16207,24 @@ packages: - supports-color dev: false + file:projects/protobuf.tgz: + resolution: {integrity: sha512-iJ4SfCFrfJTuHU+6U5u6z1V1iXRix1uBSpT6KXpT9k0g0G8HjgQSuQBYe9T5puaYfZ74HBgi5dngOZbKGAYStw==, tarball: file:projects/protobuf.tgz} + name: '@rush-temp/protobuf' + version: 0.0.0 + dependencies: + '@types/micromatch': 4.0.2 + '@types/mocha': 10.0.1 + '@types/node': 18.11.19 + c8: 7.13.0 + eslint: 8.38.0 + micromatch: 4.0.5 + mocha: 10.2.0 + rimraf: 5.0.0 + typescript: 5.0.4 + transitivePeerDependencies: + - supports-color + dev: false + file:projects/ref-doc.tgz: resolution: {integrity: sha512-deWpFwTk8amlMuVA2sry+3vGxKpyrA7Xze6W0tJIl+KZYSk+hpW1G/9LHdmW2oWzKpk/f9yMRrOQxZNyqjqDGw==, tarball: file:projects/ref-doc.tgz} name: '@rush-temp/ref-doc' @@ -16214,7 +16248,7 @@ packages: dev: false file:projects/rest.tgz: - resolution: {integrity: sha512-RaeXvup772kuLwBLyYTB4/CKzJs8CiYXVivDG/Xcw6RRml6y8W3lyoZ/U98atVZk0h6BH68YjVXsGvoSfD84Eg==, tarball: file:projects/rest.tgz} + resolution: {integrity: sha512-T68ohUIK0truidJnJ4cE97AwSiADerGRIZ/OygJQYND6gqDm8WoIyHrvQ3e3n8QMi2er7DZFOydmDJJV8qPBsg==, tarball: file:projects/rest.tgz} name: '@rush-temp/rest' version: 0.0.0 dependencies: @@ -16232,7 +16266,7 @@ packages: dev: false file:projects/samples.tgz: - resolution: {integrity: sha512-GPeX39NTlPXJ7obb2xExa91baP2iUBybR5xxSljwZaUbjf8T5SBxqGa0N7ZuIX+ozkQASOoPnnsKWL7P145ZGQ==, tarball: file:projects/samples.tgz} + resolution: {integrity: sha512-IdsJv7QvZ9yfMbzv+s+cNl0DTtwPRgYRtNdxhfm01C0rFwTz45vpRyKd0PBETz4kWbgqaNR8T/E4BbJRSgk+JQ==, tarball: file:projects/samples.tgz} name: '@rush-temp/samples' version: 0.0.0 dependencies: @@ -16322,7 +16356,7 @@ packages: dev: false file:projects/website.tgz_@types+react@18.0.34: - resolution: {integrity: sha512-jcMRDTEvDiwowlJF+U/OU2UsPJwAqS3JyH6IAR+1JqRY9B8gBV/5nQ92wIn78VaWl+MFC1eC7zS0qAct9dogCg==, tarball: file:projects/website.tgz} + resolution: {integrity: sha512-gfxpzTUoyhVb7yX9P2ZAyJnj54thZPT1qY9/0IAJFXNAkgvEHWHpCCFIVy4e3HcVuyUgTORyTcsLwZGyCCKwyg==, tarball: file:projects/website.tgz} id: file:projects/website.tgz name: '@rush-temp/website' version: 0.0.0 diff --git a/cspell.yaml b/cspell.yaml index 53f4c6b32b4..206d291b5e6 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -25,8 +25,10 @@ words: - globby - inmemory - instanceid + - interner - intrinsics - jsyaml + - keyer - lzutf - msbuild - MSRC @@ -49,6 +51,7 @@ words: - rushx - safeint - segmentof + - sfixed - strs - TRYIT - TSES diff --git a/docs/standard-library/protobuf/overview.md b/docs/standard-library/protobuf/overview.md new file mode 100644 index 00000000000..41db29ff24e --- /dev/null +++ b/docs/standard-library/protobuf/overview.md @@ -0,0 +1,233 @@ +--- +title: Overview +--- + +# The Protobuf Emitter + +TypeSpec provides an emitter (`@typespec/protobuf`) that generates Protocol Buffers specifications from TypeSpec sources as part of its standard library. The resulting Protobuf files may be used as inputs for creating gRPC services or any other tools compatible with Protocol Buffers. + +**Note**: The Protobuf emitter uses Protocol Buffers 3 (proto3) syntax. Your workflow (`protoc` version, etc.) must support proto3 to utilize this emitter. + +## Install + +In the project root, install the emitter using your JavaScript package manager of choice. For example, using NPM: + +```bash +npm install @typespec/protobuf +``` + +## Installing and enabling + +The `@typespec/protobuf` package provides an emitter that must be enabled in order to generate Protobuf files. Enable it by adding it to your TypeSpec compiler invocation on the CLI or the project configuration file: + +1. Via the CLI + +```bash +tsp compile . --emit @typespec/protobuf +``` + +2. Via the project configuration + +Add the Protobuf emitter to the `emitters` entry (or create one if it does not exist) in `tsp-project.yaml`: + +```yaml +emitters: + @typespec/protobuf: true +``` + +With this configuration entry, Protobuf files will be generated every time the project is compiled using `tsp compile .`. + +## Core concepts + +The Protobuf emitter enables you to write TypeSpec and convert it into equivalent Protocol Buffers for use with Protobuf-enabled systems (such as gRPC). Your TypeSpec models and interfaces must adhere to certain requirements and restrictions in order for the emitter to convert them to Protobuf. + +### Packages + +A protobuf package is defined by the [`TypeSpec.Protobuf.package` decorator][protobuf-package], which applies to a TypeSpec namespace. A package essentially defines a `.proto` file, and everything within the decorated namespace will be emitted to a single file. + +The following TypeSpec namespace results in a Protobuf file named `main.proto` that contains the contents of the `Test` namespace converted into Protobuf. + +```typespec +@package +namespace Test { +// ... + +} +``` + +Package names may be provided using the optional `PackageDetails` argument to the `@package` decorator. The following TypeSpec namespace will result in a file `com/example/test.proto` that has the line `package com.example.test;` within it: + +```typespec +@package({ + name: "com.example.test", +}) +namespace Test { +// ... + +} +``` + +TypeSpec objects (models, enums, etc.) are converted to Protobuf declarations within their nearest ancestor that has a package annotation. As a result, unlike in Protobuf, TypeSpec declarations of packages may be nested arbitrarily.p + +### Messages + +TypeSpec models are converted into Protobuf messages. The following TypeSpec model: + +```typespec +model TestMessage { + @field(1) n: int32; +} +``` + +will be converted into the following Protobuf message: + +```proto3 +message TestMessage { + int32 n = 1; +} +``` + +Models are converted into messages and included in the Protobuf file if any of the following conditions are met: + +- The model is explicitly annotated with the [`TypeSpec.Protobuf.message` decorator][protobuf-message]. +- The model is referenced by any service operation (see [Services](#services) below). +- The model is a direct child of a [package namespace](#packages) and has _every_ field annotated with the [`TypeSpec.Protobuf.field` decorator][protobuf-field]. + +#### Field indices + +Protobuf requires that the offset of each field within a Protobuf message be manually specified. In TypeSpec, the field indices are specified using the [`TypeSpec.Protobuf.field` decorator][protobuf-field]. All fields within a model must have an attached `@field` decorator to be converted into a Protobuf message. + +The following TypeSpec model: + +```typespec +model TestMessage { + @field(1) n: int32; +} +``` + +will be converted into the following Protobuf message: + +```proto3 +message TestMessage { + int32 n = 1; +} +``` + +### Services + +TypeSpec has a concept of a "service" defined by the [`TypeSpec.service` decorator][native-service], but the Protobuf concept of a "service" is different and is indicated by the [`TypeSpec.Protobuf.service` decorator][protobuf-service]. + +When using the Protobuf emitter, a Protobuf service designation is applied to an _interface_ within a package. For example, the following TypeSpec: + +```typespec +@package +namespace Example { + @Protobuf.service + interface Test { + // ... + } +} +``` + +will yield the following Protobuf file (named `example.proto`): + +```proto3 +syntax = "proto3"; + +package example; + +service Test { + // ... +} +``` + +#### Operations + +Within a [service interface](#services), TypeSpec operations represent Protobuf service methods. Each operation in the service interface is converted into an equivalent Protobuf method declaration. For example, the following specification: + +```typespec +model Input { + @field(1) exampleField: string; +} + +model Output { + @field(1) parsed: uint32; +} + +@Protobuf.service +interface Example { + testOperation(...Input): Output; +} +``` + +Results in the following `.proto` file: + +```proto3 +message Input { + string exampleField = 1; +} + +message Output { + uint32 parsed = 1; +} + +service Example { + rpc TestOperation(Input) returns (Output); +} +``` + +#### Streams + +The Protobuf emitter supports declaring the streaming mode of an operation using the [`TypeSpec.Protobuf.stream` decorator][protobuf-stream]. The streaming mode is specified using the [`StreamMode`][protobuf-stream-mode] enum. An operation can have one of four streaming modes: + +- `None`: this is the default mode and indicates that neither the request nor response are streamed. + + Example: `rpc Example(In) returns (Out);` + +- `In`: indicates that the request is streamed, but the response is received synchronously. + + Example: `rpc Example(stream In) returns (Out);` + +- `Out`: indicates that the request is sent synchronously, but the response is streamed. + + Example: `rpc Example(In) returns (stream Out);` + +- `Duplex`: indicates that both the request and response are streamed. + + Example: `rpc Example(stream In) returns (stream Out);` + +## Emitter options + +Emitter options can be provided to the CLI with + +```bash +--option "@typespec/protobuf.=" + +# For example +--option "@typespec/protobuf.noEmit=true" +``` + +or configured through the `typespec-project.yaml` project configuration: + +```yaml +emitters: + '@typespec/protobuf': + : + +# For example +emitters: + '@typespec/protobuf': + noEmit: true +``` + +#### `noEmit` + +If set to `true`, this emitter will not write any files. It will still validate the TypeSpec sources to ensure they are compatible with Protobuf, but the files will simply not be written to the output directory. + +[native-service]: ../built-in-decorators#service +[protobuf-service]: reference/decorators#@TypeSpec.Protobuf.service +[protobuf-package]: reference/decorators#@TypeSpec.Protobuf.package +[protobuf-field]: reference/decorators#@TypeSpec.Protobuf.field +[protobuf-stream]: reference/decorators#@TypeSpec.Protobuf.stream +[protobuf-stream-mode]: reference/data-types#TypeSpec.Protobuf.StreamMode +[protobuf-message]: reference/decorators#@TypeSpec.Protobuf.message diff --git a/docs/standard-library/protobuf/reference/data-types.md b/docs/standard-library/protobuf/reference/data-types.md new file mode 100644 index 00000000000..9fbd600810c --- /dev/null +++ b/docs/standard-library/protobuf/reference/data-types.md @@ -0,0 +1,77 @@ +--- +title: "Data types" +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# Data types + +## TypeSpec.Protobuf + +### `Extern` {#TypeSpec.Protobuf.Extern} + +A model that represents an external Protobuf reference. This type can be used to import and utilize Protobuf +declarations that are not declared in TypeSpec within TypeSpec sources. When the emitter encounters an `Extern`, it +will insert an `import` statement for the corresponding `Path` and refer to the type by `Name`. + +#### Usage + +If you have a file called `test.proto` that declares a package named `test` and a message named `Widget`, you can +use the `Extern` type to declare a model in TypeSpec that refers to your external definition of `test.Widget`. See +the example below. + +When the TypeSpec definition of `Widget` is encountered, the Protobuf emitter will represent it as a reference to +`test.Widget` and insert an import for it, rather than attempt to convert the model to an equivalent message. + +```typespec +model Extern +``` + +#### Template Parameters + +| Name | Description | +| ---- | ---------------------------------------------------------------------------------------- | +| Path | the relative path to a `.proto` file to import | +| Name | the fully-qualified reference to the type this model represents within the `.proto` file | + +### `Map` {#TypeSpec.Protobuf.Map} + +A type representing a Protobuf `map`. Instances of this type in models will be converted to the built-in `map` type +in Protobuf. + +The key type of a Protobuf `map` must be any integral type or `string`. The value type can be any type other than +another `Map`. + +```typespec +model Map +``` + +#### Template Parameters + +| Name | Description | +| ---- | ------------------------------------------------ | +| K | the key type (any integral type or string) | +| V | the value type (any type other than another map) | + +### `PackageDetails` {#TypeSpec.Protobuf.PackageDetails} + +Details applied to a package definition by the [`@package`](./decorators# + +```typespec +model TypeSpec.Protobuf.PackageDetails +``` + +### `StreamMode` {#TypeSpec.Protobuf.StreamMode} + +The streaming mode of an operation. One of: + +- `Duplex`: both the input and output of the operation are streaming. +- `In`: the input of the operation is streaming. +- `Out`: the output of the operation is streaming. +- `None`: neither the input nor the output are streaming. + +See the [`@stream`](./decorators# + +```typespec +enum TypeSpec.Protobuf.StreamMode +``` diff --git a/docs/standard-library/protobuf/reference/decorators.md b/docs/standard-library/protobuf/reference/decorators.md new file mode 100644 index 00000000000..d77a620e9d6 --- /dev/null +++ b/docs/standard-library/protobuf/reference/decorators.md @@ -0,0 +1,148 @@ +--- +title: "Decorators" +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# Decorators + +## TypeSpec.Protobuf + +### `@field` {#@TypeSpec.Protobuf.field} + +Defines the field index of a model property for conversion to a Protobuf +message. + +The field index of a Protobuf message must: + +- fall between 1 and 229 - 1, inclusive. +- not fall within the implementation reserved range of 19000 to 19999, inclusive. +- not fall within any range that was [marked reserved](# + +```typespec +dec TypeSpec.Protobuf.field(target: ModelProperty, index: uint32) +``` + +#### Target + +`ModelProperty` + +#### Parameters + +| Name | Type | Description | +| ----- | --------------- | ------------------------------------ | +| index | `scalar uint32` | The whole-number index of the field. | + +### `@message` {#@TypeSpec.Protobuf.message} + +Declares that a model is a Protobuf message. + +Messages can be detected automatically if either of the following two conditions are met: + +- The model has a `@field` annotation on all of its properties. +- The model is referenced by any service operation. + +This decorator will force the emitter to check and emit a model. + +```typespec +dec TypeSpec.Protobuf.message(target: object) +``` + +#### Target + +`model object` + +#### Parameters + +None + +### `@package` {#@TypeSpec.Protobuf.package} + +Declares that a TypeSpec namespace constitutes a Protobuf package. The contents of the namespace will be emitted to a +single Protobuf file. + +```typespec +dec TypeSpec.Protobuf.package(target: Namespace, details?: TypeSpec.Protobuf.PackageDetails) +``` + +#### Target + +`Namespace` + +#### Parameters + +| Name | Type | Description | +| ------- | ---------------------------------------- | ----------------------------------- | +| details | `model TypeSpec.Protobuf.PackageDetails` | the optional details of the package | + +### `@reserve` {#@TypeSpec.Protobuf.reserve} + +Reserve a field index, range, or name. If a field definition collides with a reservation, the emitter will produce +an error. + +This decorator accepts multiple reservations. Each reservation is one of the following: + +- a `string`, in which case the reservation refers to a field name. +- a `uint32`, in which case the reservation refers to a field index. +- a tuple `[uint32, uint32]`, in which case the reservation refers to a field range that is _inclusive_ of both ends. + +Unlike in Protobuf, where field name and index reservations must be separated, you can mix string and numeric field +reservations in a single `@reserve` call in TypeSpec. + +#### API Compatibility Note + +Field reservations prevent users of your Protobuf specification from using the given field names or indices. This can +be useful if a field is removed, as it will further prevent adding a new, incompatible field and will prevent users +from utilizing the field index at runtime in a way that may break compatibility with users of older specifications. + +See _[Protobuf Language Guide - Reserved Fields](https://protobuf.dev/programming-guides/proto3/#reserved)_ for more +information. + +```typespec +dec TypeSpec.Protobuf.reserve(target: object, ...reservations: string | [uint32, uint32] | uint32[]) +``` + +#### Target + +`model object` + +#### Parameters + +| Name | Type | Description | +| ------------ | ---------------------------------------------- | ---------------------------- | +| reservations | `model string \| [uint32, uint32] \| uint32[]` | a list of field reservations | + +### `@service` {#@TypeSpec.Protobuf.service} + +Declares that a TypeSpec interface constitutes a Protobuf service. The contents of the interface will be converted to +a `service` declaration in the resulting Protobuf file. + +```typespec +dec TypeSpec.Protobuf.service(target: Interface) +``` + +#### Target + +`Interface` + +#### Parameters + +None + +### `@stream` {#@TypeSpec.Protobuf.stream} + +Set the streaming mode of an operation. See [StreamMode](./data-types#TypeSpec.Protobuf.StreamMode) for more information. + +```typespec +dec TypeSpec.Protobuf.stream(target: Operation, mode: TypeSpec.Protobuf.StreamMode) +``` + +#### Target + +`Operation` + +#### Parameters + +| Name | Type | Description | +| ---- | ----------------------------------- | ---------------------------------------------- | +| mode | `enum TypeSpec.Protobuf.StreamMode` | The streaming mode to apply to this operation. | diff --git a/docs/standard-library/protobuf/reference/index.md b/docs/standard-library/protobuf/reference/index.md new file mode 100644 index 00000000000..e39c80cfdc1 --- /dev/null +++ b/docs/standard-library/protobuf/reference/index.md @@ -0,0 +1,23 @@ +--- +title: Index +sidebar_position: 0 +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +## TypeSpec.Protobuf + +### Decorators + +- [`@field`](./decorators.md#@TypeSpec.Protobuf.field) +- [`@message`](./decorators.md#@TypeSpec.Protobuf.message) +- [`@package`](./decorators.md#@TypeSpec.Protobuf.package) +- [`@reserve`](./decorators.md#@TypeSpec.Protobuf.reserve) +- [`@service`](./decorators.md#@TypeSpec.Protobuf.service) +- [`@stream`](./decorators.md#@TypeSpec.Protobuf.stream) + +### Models + +- [`Extern`](./data-types.md#TypeSpec.Protobuf.Extern) +- [`Map`](./data-types.md#TypeSpec.Protobuf.Map) +- [`PackageDetails`](./data-types.md#TypeSpec.Protobuf.PackageDetails) diff --git a/packages/protobuf/.c8rc.json b/packages/protobuf/.c8rc.json new file mode 100644 index 00000000000..df1397419db --- /dev/null +++ b/packages/protobuf/.c8rc.json @@ -0,0 +1,3 @@ +{ + "reporter": ["cobertura", "json", "text"] +} diff --git a/packages/protobuf/.eslintrc.cjs b/packages/protobuf/.eslintrc.cjs new file mode 100644 index 00000000000..c0b2a9d1a75 --- /dev/null +++ b/packages/protobuf/.eslintrc.cjs @@ -0,0 +1,7 @@ +require("@typespec/eslint-config-typespec/patch/modern-module-resolution"); + +module.exports = { + plugins: ["@typespec/eslint-plugin"], + extends: ["@typespec/eslint-config-typespec", "plugin:@typespec/eslint-plugin/recommended"], + parserOptions: { tsconfigRootDir: __dirname }, +}; diff --git a/packages/protobuf/.gitignore b/packages/protobuf/.gitignore new file mode 100644 index 00000000000..bea41e259d6 --- /dev/null +++ b/packages/protobuf/.gitignore @@ -0,0 +1 @@ +.protoc-out diff --git a/packages/protobuf/.mocharc.yaml b/packages/protobuf/.mocharc.yaml new file mode 100644 index 00000000000..2c757438b54 --- /dev/null +++ b/packages/protobuf/.mocharc.yaml @@ -0,0 +1,4 @@ +timeout: 5000 +require: source-map-support/register +spec: "dist/test/**/*.js" +ignore: "dist/test/manual/**/*.js" diff --git a/packages/protobuf/CHANGELOG.json b/packages/protobuf/CHANGELOG.json new file mode 100644 index 00000000000..8085c132bc8 --- /dev/null +++ b/packages/protobuf/CHANGELOG.json @@ -0,0 +1,4 @@ +{ + "name": "@typespec/protobuf", + "entries": [] +} diff --git a/packages/protobuf/LICENSE b/packages/protobuf/LICENSE new file mode 100644 index 00000000000..21071075c24 --- /dev/null +++ b/packages/protobuf/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/packages/protobuf/README.md b/packages/protobuf/README.md new file mode 100644 index 00000000000..5143cc0445c --- /dev/null +++ b/packages/protobuf/README.md @@ -0,0 +1,30 @@ +# TypeSpec Protobuf library and emitter + +**Warning** :warning:: This pre-release emitter is in active development and may change. Your feedback is highly appreciated. + +This package provides support for defining and emitting Protobuf specifications in [TypeSpec](https://github.com/microsoft/typespec) and an emitter that generates Protobuf output files from TypeSpec sources. + +## Install + +In your project root: + +```bash +npm install @typespec/protobuf +``` + +## Using the Protobuf emitter + +1. Using the TypeSpec CLI (`tsp`): + +```bash +tsp compile . --emit @typespec/protobuf +``` + +2. Using the TypeSpec project configuration file: + +Add the following to your `tspproject.yaml` file. + +```yaml +emit: + - "@typespec/protobuf" +``` diff --git a/packages/protobuf/lib/proto.tsp b/packages/protobuf/lib/proto.tsp new file mode 100644 index 00000000000..bea96e092ae --- /dev/null +++ b/packages/protobuf/lib/proto.tsp @@ -0,0 +1,296 @@ +import "../dist/src/proto.js"; + +namespace TypeSpec.Protobuf; + +/** + * A model that represents an external Protobuf reference. This type can be used to import and utilize Protobuf + * declarations that are not declared in TypeSpec within TypeSpec sources. When the emitter encounters an `Extern`, it + * will insert an `import` statement for the corresponding `Path` and refer to the type by `Name`. + * + * #### Usage + * + * If you have a file called `test.proto` that declares a package named `test` and a message named `Widget`, you can + * use the `Extern` type to declare a model in TypeSpec that refers to your external definition of `test.Widget`. See + * the example below. + * + * When the TypeSpec definition of `Widget` is encountered, the Protobuf emitter will represent it as a reference to + * `test.Widget` and insert an import for it, rather than attempt to convert the model to an equivalent message. + * + * @template Path the relative path to a `.proto` file to import + * @template Name the fully-qualified reference to the type this model represents within the `.proto` file + * + * @example + * + * ```typespec + * model Widget is Extern<"path/to/test.proto", "test.Widget">; + * ``` + */ +@externRef(Path, Name) +model Extern { + // This _extern property is needed so that getEffectiveModelType will have something to look up. Without it, if an + // Extern model is spread into the parameter of an operation, the resulting model is empty and carries no information + // that can relate it back to its original definition. + _extern: never; +} + +/** + * Contains some common well-known Protobuf types defined by the google.protobuf library. + */ +namespace WellKnown { + /** + * An empty message. + * + * This model references `google.protobuf.Empty` from `google/protobuf/empty.proto`. + */ + model Empty is Extern<"google/protobuf/empty.proto", "google.protobuf.Empty">; + + /** + * A timestamp. + * + * This model references `google.protobuf.Timestamp` from `google/protobuf/timestamp.proto`. + */ + model Timestamp is Extern<"google/protobuf/timestamp.proto", "google.protobuf.Timestamp">; + + /** + * Any value. + * + * This model references `google.protobuf.Any` from `google/protobuf/any.proto`. + */ + model Any is Extern<"google/protobuf/any.proto", "google.protobuf.Any">; +} + +/** + * A signed 32-bit integer that will use the `sint32` encoding when used in a Protobuf message. + * + * #### Protobuf binary format + * + * Uses variable-length encoding. These more efficiently encode negative numbers than regular int32s. + */ +scalar sint32 extends int32; +/** + * A signed 64-bit integer that will use the `sint64` encoding when used in a Protobuf message. + * + * #### Protobuf binary format + * + * Uses variable-length encoding. These more efficiently encode negative numbers than regular `int64s`. + */ +scalar sint64 extends int64; +/** + * A signed 32-bit integer that will use the `sfixed32` encoding when used in a Protobuf message. + * + * #### Protobuf binary format + * + * Always four bytes. + */ +scalar sfixed32 extends int32; +/** + * A signed 64-bit integer that will use the `sfixed64` encoding when used in a Protobuf message. + * + * #### Protobuf binary format + * + * Always eight bytes. + */ +scalar sfixed64 extends int64; +/** + * An unsigned 32-bit integer that will use the `fixed32` encoding when used in a Protobuf message. + * + * #### Protobuf binary format + * + * Always four bytes. More efficient than `uint32` if values are often greater than 228. + */ +scalar fixed32 extends uint32; +/** + * An unsigned 64-bit integer that will use the `fixed64` encoding when used in a Protobuf message. + * + * #### Protobuf binary format + * + * Always eight bytes. More efficient than `uint64` if values are often greater than 256. + */ +scalar fixed64 extends uint64; + +/** + * Types recognized as "integral" types + */ +alias integral = int32 | int64 | uint32 | uint64 | boolean; + +/** + * A type representing a Protobuf `map`. Instances of this type in models will be converted to the built-in `map` type + * in Protobuf. + * + * The key type of a Protobuf `map` must be any integral type or `string`. The value type can be any type other than + * another `Map`. + * + * @template K the key type (any integral type or string) + * @template V the value type (any type other than another map) + */ +@_map +model Map {} + +/** + * Declares that a model is a Protobuf message. + * + * Messages can be detected automatically if either of the following two conditions are met: + * + * - The model has a `@field` annotation on all of its properties. + * - The model is referenced by any service operation. + * + * This decorator will force the emitter to check and emit a model. + */ +extern dec message(target: object); + +/** + * Defines the field index of a model property for conversion to a Protobuf + * message. + * + * The field index of a Protobuf message must: + * - fall between 1 and 229 - 1, inclusive. + * - not fall within the implementation reserved range of 19000 to 19999, inclusive. + * - not fall within any range that was [marked reserved](#@TypeSpec.Protobuf.reserve). + * + * #### API Compatibility Note + * + * Fields are accessed by index, so changing the index of a field is an API breaking change. + * + * #### Encoding + * + * Field indices between 1 and 15 are encoded using a single byte, while field indices from 16 through 2047 require two + * bytes, so those indices between 1 and 15 should be preferred and reserved for elements that are frequently or always + * set in the message. See the [Protobuf binary format](https://protobuf.dev/programming-guides/encoding/). + * + * @param index The whole-number index of the field. + * + * @example + * + * ```typespec + * model ExampleMessage { + * @field(1) + * test: string; + * } + * ``` + */ +extern dec field(target: TypeSpec.Reflection.ModelProperty, index: uint32); + +/** + * Reserve a field index, range, or name. If a field definition collides with a reservation, the emitter will produce + * an error. + * + * This decorator accepts multiple reservations. Each reservation is one of the following: + * + * - a `string`, in which case the reservation refers to a field name. + * - a `uint32`, in which case the reservation refers to a field index. + * - a tuple `[uint32, uint32]`, in which case the reservation refers to a field range that is _inclusive_ of both ends. + * + * Unlike in Protobuf, where field name and index reservations must be separated, you can mix string and numeric field + * reservations in a single `@reserve` call in TypeSpec. + * + * #### API Compatibility Note + * + * Field reservations prevent users of your Protobuf specification from using the given field names or indices. This can + * be useful if a field is removed, as it will further prevent adding a new, incompatible field and will prevent users + * from utilizing the field index at runtime in a way that may break compatibility with users of older specifications. + * + * See _[Protobuf Language Guide - Reserved Fields](https://protobuf.dev/programming-guides/proto3/#reserved)_ for more + * information. + * + * @param reservations a list of field reservations + * + * @example + * + * ```typespec + * // Reserve the fields 8-15 inclusive, 100, and the field name "test" within a model. + * @reserve([8, 15], 100, "test") + * model Example { + * // ... + * } + * ``` + */ +extern dec reserve(target: object, ...reservations: (string | [uint32, uint32] | uint32)[]); + +/** + * Declares that a TypeSpec interface constitutes a Protobuf service. The contents of the interface will be converted to + * a `service` declaration in the resulting Protobuf file. + */ +extern dec service(target: TypeSpec.Reflection.Interface); + +// FIXME: cannot link to the package decorator directly because it is detected as a broken link. +/** + * Details applied to a package definition by the [`@package`](./decorators#@TypeSpec.Protobuf.package) decorator. + */ +model PackageDetails { + /** + * The package's name. + * + * By default, the package's name is constructed from the namespace it is applied to. + */ + name?: string; + /** + * The package's top-level options. + * + * See the [Protobuf Language Guide - Options](https://protobuf.dev/programming-guides/proto3/#options) for more information. + * + * Currently, only string, boolean, and numeric options are supported. + */ + options?: Record; +} + +/** + * Declares that a TypeSpec namespace constitutes a Protobuf package. The contents of the namespace will be emitted to a + * single Protobuf file. + * + * @param details the optional details of the package + */ +extern dec package(target: TypeSpec.Reflection.Namespace, details?: PackageDetails); + +/** + * The streaming mode of an operation. One of: + * + * - `Duplex`: both the input and output of the operation are streaming. + * - `In`: the input of the operation is streaming. + * - `Out`: the output of the operation is streaming. + * - `None`: neither the input nor the output are streaming. + * + * See the [`@stream`](./decorators#@TypeSpec.Protobuf.stream) decorator. + */ +enum StreamMode { + /** + * Both the input and output of the operation are streaming. Both the client and service will stream messages to each + * other until the connections are closed. + */ + Duplex, + /** + * The input of the operation is streaming. The client will send a stream of events; and, once the stream is closed, + * the service will respond with a message. + */ + In, + /** + * The output of the operation is streaming. The client will send a message to the service, and the service will send + * a stream of events back to the client. + */ + Out, + /** + * Neither the input nor the output are streaming. This is the default mode of an operation without the `@stream` + * decorator. + */ + None, +} + +/** + * Set the streaming mode of an operation. See [StreamMode](./data-types#TypeSpec.Protobuf.StreamMode) for more information. + * + * @param mode The streaming mode to apply to this operation. + * + * @example + * + * ```typespec + * @stream(StreamMode.Out) + * op logs(...LogsRequest): LogEvent; + * ``` + * + * @example + * + * ```typespec + * @stream(StreamMode.Duplex) + * op connectToMessageService(...Message): Message; + * ``` + */ +extern dec stream(target: TypeSpec.Reflection.Operation, mode: StreamMode); diff --git a/packages/protobuf/package.json b/packages/protobuf/package.json new file mode 100644 index 00000000000..39fb2915f4e --- /dev/null +++ b/packages/protobuf/package.json @@ -0,0 +1,49 @@ +{ + "name": "@typespec/protobuf", + "version": "0.43.0", + "author": "Microsoft Corporation", + "description": "TypeSpec library and emitter for Protobuf (gRPC)", + "homepage": "https://github.com/Microsoft/typespec", + "readme": "https://github.com/Microsoft/typespec/blob/main/packages/protobuf/README.md", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/Microsoft/typespec.git" + }, + "bugs": { + "url": "https://github.com/Microsoft/typespec/issues" + }, + "keywords": [ + "typespec", + "protobuf", + "grpc" + ], + "main": "dist/src/lib.js", + "type": "module", + "tspMain": "lib/proto.tsp", + "scripts": { + "clean": "rimraf ./dist ./temp", + "build": "tsc -p .", + "watch": "tsc -p . --watch", + "test": "mocha", + "test-official": "c8 mocha --forbid-only", + "lint": "eslint . --ext .ts --max-warnings=0", + "lint:fix": "eslint . --fix --ext .ts" + }, + "dependencies": { + "@typespec/compiler": "~0.43.0" + }, + "devDependencies": { + "@typespec/eslint-config-typespec": "~0.6.0", + "@typespec/eslint-plugin": "~0.43.0", + "@types/mocha": "~10.0.0", + "@types/node": "~18.11.9", + "c8": "~7.13.0", + "eslint": "^8.36.0", + "mocha": "~10.2.0", + "rimraf": "~5.0.0", + "typescript": "~5.0.2", + "micromatch": "^4.0.5", + "@types/micromatch": "^4.0.2" + } +} diff --git a/packages/protobuf/src/ast.ts b/packages/protobuf/src/ast.ts new file mode 100644 index 00000000000..c43456893db --- /dev/null +++ b/packages/protobuf/src/ast.ts @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft Corporation. + +import { Namespace } from "@typespec/compiler"; + +/** + * This module describes an AST for Protobuf. + */ + +/** + * A single .proto file. + */ +export interface ProtoFile { + /** + * The package name, if one is known. + */ + package?: string; + /** + * The `option` specifiers to include in the file. + */ + options: Partial; + + /** + * Paths imported by this file. + */ + imports: string[]; + + /** + * The declarations in the file. + * + * Only `service` and `message` declarations may exist at the root of the file. + */ + declarations: Iterable; + + /** + * The original namespace node from which this ProtoFile originated. + */ + source: Namespace; +} + +/** + * The built-in options that are defined by Protobuf. + */ +export interface WellKnownFileOptions { + java_package: string; + java_outer_classname: string; + + optimize_for: "SPEED" | "CODE_SIZE" | "LITE_RUNTIME"; + + cc_enable_arenas: boolean; +} + +/** + * A top-level declaration. + */ +export type ProtoTopLevelDeclaration = + | ProtoServiceDeclaration + | ProtoMessageDeclaration + | ProtoEnumDeclaration; + +/** + * A declaration. One of `service`, `message`, a field within a message, `one_of`, `enum`, or an `rpc` method. + */ +export type ProtoDeclaration = + | ProtoServiceDeclaration + | ProtoMessageDeclaration + | ProtoFieldDeclaration + | ProtoOneOfDeclaration + | ProtoEnumDeclaration + | ProtoMethodDeclaration; + +/** + * A Protobuf scalar type. + */ +export type ScalarName = ScalarIntegralName | "double" | "float" | "bytes" | "string"; + +/** + * A Protobuf integral type. + */ +export type ScalarIntegralName = ScalarIntegerName | ScalarFixedName | "bool"; + +/** + * A Protobuf variable-length integer type. + */ +export type ScalarIntegerName = `${"u" | "s" | ""}int${"32" | "64"}`; + +/** + * A Protobuf fixed-length integer type. + */ +export type ScalarFixedName = `${"s" | ""}fixed${"32" | "64"}`; + +// Symbols for type destructuring +const $scalar = Symbol("$scalar"); +const $ref = Symbol("$ref"); +const $map = Symbol("$map"); + +/** + * A map type. Map keys can be any integral or string type (any scalar except float, double, and bytes). + * + * The value may be any type other than another map. + */ +export type ProtoMap = [typeof $map, ScalarIntegralName | "string", ProtoRef | ProtoScalar]; + +/** + * A reference to a named message type. + */ +export type ProtoRef = [typeof $ref, string]; + +/** + * A scalar type. + */ +export type ProtoScalar = [typeof $scalar, ScalarName]; + +/** + * A Protobuf type. + */ +export type ProtoType = ProtoScalar | ProtoRef | ProtoMap; + +/** + * Create a scalar type by name. + */ +export function scalar(t: ScalarName): ProtoScalar { + return [$scalar, t]; +} + +/** + * Create a type reference (symbol) to a named message. + */ +export function ref(t: string): ProtoRef { + return [$ref, t]; +} + +/** + * Create a map from a key type to a value type. + */ +export function map(k: ScalarIntegralName | "string", v: Exclude): ProtoMap { + return [$map, k, v]; +} + +/* c8 ignore start */ + +// Unreachable, by definition, should not be covered :) + +/** + * Creates a type that will throw an internal error if the system attempts to emit it. + * + * @param message - optional message that should be printed + */ +export function unreachable(message: string = "tried to emit unreachable type"): never { + // This little "array-like" object will throw an internal error as soon as the "tag" is inspected. + return Object.freeze({ + get [0]() { + throw new Error("Internal Error: " + message); + }, + }) as never; +} + +/* c8 ignore stop */ + +/** + * A "pattern" object with variants for each Protobuf type. + */ +export interface ProtoTypeMatchPattern { + scalar: (s: ScalarName) => T; + ref: (r: string) => T; + map: (k: ScalarIntegralName | "string", v: Exclude) => T; +} + +/** + * A helper function that matches and delegates a Protobuf type to a handler per type. + * + * @param type - the Protobuf type to match and delegate + * @param pattern - the matching pattern of delegates to apply + * @returns + */ +export function matchType(type: ProtoType, pattern: ProtoTypeMatchPattern): Result { + switch (type[0]) { + case $ref: + return pattern.ref(type[1]); + case $scalar: + return pattern.scalar(type[1]); + case $map: + return pattern.map(type[1], type[2] as Exclude); + /* c8 ignore next 5 */ + default: + const __exhaust: never = type[0]; + throw new Error(`Internal Error: unreachable matchType variant ${__exhaust}`); + } +} + +/** + * A `service` declaration. + */ +export interface ProtoServiceDeclaration { + kind: "service"; + name: string; + operations: ProtoMethodDeclaration[]; +} + +/** + * An operation's streaming mode. + */ +export const enum StreamingMode { + Duplex = 3, + In = 2, + Out = 1, + None = 0, +} + +/** + * An `rfc` method declaration. + */ +export interface ProtoMethodDeclaration { + kind: "method"; + stream: StreamingMode; + name: string; + input: ProtoRef; + returns: ProtoRef; +} + +/** + * A declaration that can fit within the body of a message declaration. + */ +export type ProtoMessageBodyDeclaration = + | ProtoFieldDeclaration + | ProtoMessageDeclaration + | ProtoOneOfDeclaration + | ProtoEnumDeclaration; + +/** + * A `message` declaration. + */ +export interface ProtoMessageDeclaration { + kind: "message"; + name: string; + declarations: Array; + reservations?: Array; +} + +/** + * A field declaration within a message. + */ +export interface ProtoFieldDeclaration { + kind: "field"; + name: string; + /** + * Whether or not the field is repeated (i.e. an array). + */ + repeated?: boolean; + options?: Partial; + type: ProtoType; + index: number; +} + +/** + * The options for fields defined by the Protobuf specification. + */ +export interface DefaultFieldOptions { + packed: true; + deprecated: true; +} + +/** + * A `one_of` declaration. + */ +export interface ProtoOneOfDeclaration { + kind: "oneof"; + name: string; + declarations: ProtoFieldDeclaration[]; +} + +/** + * An `enum` declaration. + */ +export interface ProtoEnumDeclaration { + kind: "enum"; + name: string; + allowAlias?: boolean; + variants: [string, number][]; +} diff --git a/packages/protobuf/src/lib.ts b/packages/protobuf/src/lib.ts new file mode 100644 index 00000000000..bbf1d936fff --- /dev/null +++ b/packages/protobuf/src/lib.ts @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { createTypeSpecLibrary, JSONSchemaType, paramMessage } from "@typespec/compiler"; + +/** + * Options that the Protobuf emitter accepts. + */ +export interface ProtobufEmitterOptions { + /** + * Don't emit anything. + */ + noEmit?: boolean; +} + +const EmitterOptionsSchema: JSONSchemaType = { + type: "object", + additionalProperties: false, + properties: { + noEmit: { type: "boolean", nullable: true }, + }, + required: [], +}; + +const PACKAGE_NAME = "@typespec/protobuf"; + +export const TypeSpecProtobufLibrary = createTypeSpecLibrary({ + name: PACKAGE_NAME, + requireImports: [PACKAGE_NAME], + diagnostics: { + "field-index": { + severity: "error", + messages: { + missing: paramMessage`field ${"name"} does not have a field index, but one is required (try using the '@field' decorator)`, + invalid: paramMessage`field index ${"index"} is invalid (must be an integer greater than zero)`, + "out-of-bounds": paramMessage`field index ${"index"} is out of bounds (must be less than ${"max"})`, + reserved: paramMessage`field index ${"index"} falls within the implementation-reserved range of 19000-19999 inclusive`, + "user-reserved": paramMessage`field index ${"index"} was reserved by a call to @reserve on this model`, + "user-reserved-range": paramMessage`field index ${"index"} falls within a range reserved by a call to @reserve on this model`, + }, + }, + "field-name": { + severity: "error", + messages: { + "user-reserved": paramMessage`field name '${"name"}' was reserved by a call to @reserve on this model`, + }, + }, + "root-operation": { + severity: "error", + messages: { + default: + "operations in the root namespace are not supported (no associated Protobuf service)", + }, + }, + "unsupported-intrinsic": { + severity: "error", + messages: { + default: paramMessage`intrinsic type ${"name"} is not supported in Protobuf`, + }, + }, + "unsupported-return-type": { + severity: "error", + messages: { + default: "Protobuf methods must return a named Model", + }, + }, + "unsupported-input-type": { + severity: "error", + messages: { + "wrong-number": + "Protobuf methods must accept exactly one Model input (an empty model will do)", + "wrong-type": "Protobuf methods may only accept a named Model as an input", + unconvertible: "input parameters cannot be converted to a Protobuf message", + }, + }, + "unsupported-field-type": { + severity: "error", + messages: { + unconvertible: paramMessage`cannot convert a ${"type"} to a protobuf type (only intrinsic types and models are supported)`, + "unknown-intrinsic": paramMessage`no known protobuf scalar for intrinsic type ${"name"}`, + "unknown-scalar": paramMessage`no known protobuf scalar for TypeSpec scalar type ${"name"}`, + "recursive-map": "a protobuf map's 'value' type may not refer to another map", + union: "a message field's type may not be a union", + }, + }, + "namespace-collision": { + severity: "error", + messages: { + default: paramMessage`the package name ${"name"} has already been used`, + }, + }, + "unconvertible-enum": { + severity: "error", + messages: { + default: + "enums must explicitly assign exactly one integer to each member to be used in a Protobuf message", + "no-zero-first": + "the first variant of an enum must be set to zero to be used in a Protobuf message", + }, + }, + "nested-array": { + severity: "error", + messages: { + default: "nested arrays are not supported by the Protobuf emitter", + }, + }, + "invalid-package-name": { + severity: "error", + messages: { + default: paramMessage`${"name"} is not a valid package name (must consist of letters and numbers separated by ".")`, + }, + }, + "illegal-reservation": { + severity: "error", + messages: { + default: + "reservation value must be a string literal, uint32 literal, or a tuple of two uint32 literals denoting a range", + }, + }, + "model-not-in-package": { + severity: "error", + messages: { + default: paramMessage`model ${"name"} is not in a namespace that uses the '@Protobuf.package' decorator`, + }, + }, + "anonymous-model": { + severity: "error", + messages: { + default: "anonymous models cannot be used in Protobuf messages", + }, + }, + package: { + severity: "error", + messages: { + "disallowed-option-type": paramMessage`option '${"name"}' with type '${"type"}' is not allowed in a package declaration (only string, boolean, and numeric types are allowed)`, + }, + }, + }, + emitter: { options: EmitterOptionsSchema }, +}); + +export const { reportDiagnostic } = TypeSpecProtobufLibrary; + +export { $onEmit } from "./proto.js"; + +export type TypeSpecProtobufLibrary = typeof TypeSpecProtobufLibrary; + +const keys = [ + "fieldIndex", + "package", + "service", + "externRef", + "stream", + "reserve", + "message", + "_map", +] as const; + +export const state = Object.fromEntries( + keys.map((k) => [k, TypeSpecProtobufLibrary.createStateSymbol(k)]) +) as { + [K in (typeof keys)[number]]: symbol; +}; diff --git a/packages/protobuf/src/proto.ts b/packages/protobuf/src/proto.ts new file mode 100644 index 00000000000..3e17d03566d --- /dev/null +++ b/packages/protobuf/src/proto.ts @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + DecoratorContext, + EmitContext, + EmitOptionsFor, + EnumMember, + Interface, + Model, + ModelProperty, + Namespace, + NumericLiteral, + Operation, + Program, + resolvePath, + Tuple, + Type, +} from "@typespec/compiler"; + +import { StreamingMode } from "./ast.js"; +import { ProtobufEmitterOptions, reportDiagnostic, state, TypeSpecProtobufLibrary } from "./lib.js"; +import { createProtobufEmitter } from "./transform/index.js"; + +/** + * # @typespec/protobuf : Protobuf/gRPC Emitter and Decorators for TypeSpec + * + * This module defines an emitter and decorator library for TypeSpec that enables specifying Protobuf services and models. + */ + +/** + * The maximum field index allowed by Protocol Buffers. + */ +const MAX_FIELD_INDEX = 2 ** 29 - 1; + +/** + * The field range between 19000 and 19999 is reserved for Protobuf client implementations. + */ +const IMPLEMENTATION_RESERVED_RANGE = [19000, 19999] as const; + +/** + * Defined in the [ProtoBuf Language Spec](https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#identifiers). + * + * ident = letter { letter | decimalDigit | "_" } + * fullIdent = ident { "." ident } + */ +export const PROTO_FULL_IDENT = /([a-zA-Z][a-zA-Z0-9_]*)+/; + +/** + * Decorate an interface as a service, indicating that it represents a Protobuf `service` declaration. + * + * @param ctx - decorator context + * @param target - the decorated interface + */ +export function $service(ctx: DecoratorContext, target: Interface) { + ctx.program.stateSet(state.service).add(target); +} + +export interface PackageDetails { + name?: string; +} + +/** + * Declare a Protobuf package. + * + * @param ctx - decorator context + * @param target - target decorator namespace + */ +export function $package(ctx: DecoratorContext, target: Namespace, details?: Model) { + ctx.program.stateMap(state.package).set(target, details); +} + +/** + * Determines whether a type represents a Protobuf map. + * + * @param program - the program context + * @param m - the type to test + * @returns true if the internal representation of a Protobuf map is bound to this type. + */ +export function isMap(program: Program, m: Type): boolean { + return program.stateSet(state._map).has(m); +} + +/** + * Binds the internal representation of a Protobuf map. + * @internal + * @param ctx + * @param target + */ +export function $_map(ctx: DecoratorContext, target: Model) { + ctx.program.stateSet(state._map).add(target); +} + +export function $externRef(ctx: DecoratorContext, target: Model, path: string, name: string) { + ctx.program.stateMap(state.externRef).set(target, [path, name]); +} + +export function $stream(ctx: DecoratorContext, target: Operation, mode: EnumMember) { + const emitStreamingMode = { + Duplex: StreamingMode.Duplex, + In: StreamingMode.In, + Out: StreamingMode.Out, + None: StreamingMode.None, + }[mode.name as string]; + + ctx.program.stateMap(state.stream).set(target, emitStreamingMode); +} + +function getTuple(program: Program, t: Type): [number, number] | null { + if (t.kind !== "Tuple" || t.values.some((v) => v.kind !== "Number") || t.values.length !== 2) { + reportDiagnostic(program, { + code: "illegal-reservation", + target: t, + }); + + return null; + } + + return Object.assign( + (t as Tuple).values.map((v) => (v as NumericLiteral).value) as [number, number], + { type: t } + ); +} + +export type Reservation = string | number | ([number, number] & { type: Type }); + +export function $reserve( + ctx: DecoratorContext, + target: Model, + ...reservations: readonly (Type | number | string)[] +) { + const finalReservations = reservations + .map((reservation) => + typeof reservation === "object" ? getTuple(ctx.program, reservation) : reservation + ) + .filter((v) => v != null); + + ctx.program.stateMap(state.reserve).set(target, finalReservations); +} + +export function $message(ctx: DecoratorContext, target: Model) { + ctx.program.stateSet(state.message).add(target); +} + +/** + * Decorate a model property with a field index. Field indices are required for all fields of emitted messages. + * + * @param param0 + * @param target + * @param fieldIndex + * @returns + */ +export function $field(ctx: DecoratorContext, target: ModelProperty, fieldIndex: number) { + if (!Number.isInteger(fieldIndex) || fieldIndex <= 0) { + reportDiagnostic(ctx.program, { + code: "field-index", + messageId: "invalid", + format: { + index: String(fieldIndex), + }, + target, + }); + return; + } else if (fieldIndex > MAX_FIELD_INDEX) { + reportDiagnostic(ctx.program, { + code: "field-index", + messageId: "out-of-bounds", + format: { + index: String(fieldIndex), + max: String(MAX_FIELD_INDEX + 1), + }, + target, + }); + return; + } else if ( + fieldIndex >= IMPLEMENTATION_RESERVED_RANGE[0] && + fieldIndex <= IMPLEMENTATION_RESERVED_RANGE[1] + ) { + reportDiagnostic(ctx.program, { + code: "field-index", + messageId: "reserved", + format: { + index: String(fieldIndex), + }, + target, + }); + } + + ctx.program.stateMap(state.fieldIndex).set(target, fieldIndex); +} + +/** + * Emitter main function. + * + * @param program - the program to emit + */ +export async function $onEmit(ctx: EmitContext>) { + const emitter = createProtobufEmitter(ctx.program); + + await emitter(resolvePath(ctx.emitterOutputDir), ctx.options); +} + +/** + * Validation function + */ +export async function $onValidate(program: Program) { + // Is this correct? See https://github.com/microsoft/typespec/issues/1859 + /* c8 ignore next 6 */ + if (program.compilerOptions.noEmit) { + const options = program.emitters.find((e) => e.emitFunction === $onEmit) + ?.options as ProtobufEmitterOptions; + const emitter = createProtobufEmitter(program); + await emitter("", options); + } +} + +export const namespace = "TypeSpec.Protobuf"; +export { TypeSpecProtobufLibrary as $lib }; diff --git a/packages/protobuf/src/transform/index.ts b/packages/protobuf/src/transform/index.ts new file mode 100644 index 00000000000..121febbe620 --- /dev/null +++ b/packages/protobuf/src/transform/index.ts @@ -0,0 +1,981 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + DiagnosticTarget, + Enum, + formatDiagnostic, + getEffectiveModelType, + getTypeName, + Interface, + IntrinsicType, + isDeclaredInNamespace, + Model, + ModelProperty, + Namespace, + Operation, + Program, + resolvePath, + Scalar, + StringLiteral, + SyntaxKind, + Type, + Union, +} from "@typespec/compiler"; +import { EOL } from "os"; +import { + map, + matchType, + ProtoEnumDeclaration, + ProtoFieldDeclaration, + ProtoFile, + ProtoMap, + ProtoMessageBodyDeclaration, + ProtoMessageDeclaration, + ProtoMethodDeclaration, + ProtoRef, + ProtoScalar, + ProtoTopLevelDeclaration, + ProtoType, + ref, + scalar, + ScalarIntegralName, + StreamingMode, + unreachable, +} from "../ast.js"; +import { ProtobufEmitterOptions, reportDiagnostic, state } from "../lib.js"; +import { $field, isMap, Reservation } from "../proto.js"; +import { writeProtoFile } from "../write.js"; + +// Cache for scalar -> ProtoScalar map +const _protoScalarsMap = new WeakMap>(); +const _protoExternMap = new WeakMap>(); + +/** + * Create a worker function that converts the TypeSpec program to Protobuf and writes it to the file system. + */ +export function createProtobufEmitter( + program: Program +): (outDir: string, options: ProtobufEmitterOptions) => Promise { + return async function doEmit(outDir, options) { + // Convert the program to a set of proto files. + const files = tspToProto(program); + + if (!program.compilerOptions.noEmit && !options?.noEmit && !program.hasError()) { + for (const file of files) { + // If the file has a package, emit it to a path that is shaped like the package name. Otherwise emit to + // main.proto + + // Collisions have already been detected. + + const packageSlug = file.package?.split(".") ?? ["main"]; + const filePath = resolvePath(outDir, ...packageSlug.slice(0, -1)); + + await program.host.mkdirp(filePath); + await program.host.writeFile( + resolvePath(filePath, packageSlug[packageSlug.length - 1] + ".proto"), + writeProtoFile(file) + ); + } + } + }; +} + +/** + * Create a set of proto files that represent the TypeSpec program. + * + * This is the meat of the emitter. + */ +function tspToProto(program: Program): ProtoFile[] { + const packages = new Set( + program.stateMap(state.package).keys() as Iterable + ); + + const serviceInterfaces = [...(program.stateSet(state.service) as Set)]; + + const declarationMap = new Map( + [...packages].map((p) => [p, []]) + ); + + const visitedTypes = new Set(); + + /** + * Visits a model type, converting it into a message definition and adding it if it has not already been visited. + * @param model - the model type to consider + */ + function visitModel(model: Model, source: Type) { + const modelPackage = getPackageOfType(program, model); + const declarations = modelPackage && declarationMap.get(modelPackage); + + if (!declarations) { + reportDiagnostic(program, { + target: source, + code: "model-not-in-package", + format: { name: model.name }, + }); + } + + if (!visitedTypes.has(model)) { + visitedTypes.add(model); + declarations?.push(toMessage(model)); + } + } + + /** + * Visits an enum type, converting it into a Protobuf enum definition and adding it if it has not already been visited. + */ + function visitEnum(e: Enum) { + const modelPackage = getPackageOfType(program, e); + const declarations = modelPackage && declarationMap.get(modelPackage); + if (!visitedTypes.has(e)) { + visitedTypes.add(e); + + const members = [...e.members.values()]; + + // We only support enums where every variant is explicitly assigned an integer value + if ( + members.some( + ({ value: v }) => v === undefined || typeof v !== "number" || !Number.isInteger(v) + ) + ) { + reportDiagnostic(program, { + target: e, + code: "unconvertible-enum", + }); + } + + // we also only support enums where the first value is zero. + if (members[0].value !== 0) { + reportDiagnostic(program, { + target: members[0], + code: "unconvertible-enum", + messageId: "no-zero-first", + }); + } + + declarations?.push(toEnum(e)); + } + } + + const importMap = new Map([...packages].map((ns) => [ns, new Set()])); + + function typeWantsImport(program: Program, t: Model | Operation, path: string) { + const packageNs = getPackageOfType(program, t); + + if (packageNs) { + importMap.get(packageNs)?.add(path); + } + } + + const mapImportSourceInformation = new WeakMap< + ProtoMap, + [Model | Operation, NamespaceTraversable] + >(); + + const effectiveModelCache = new Map(); + + for (const packageNs of packages) { + addDeclarationsOfPackage(packageNs); + } + + // Emit a file per package. + const files = [...packages].map((namespace) => { + const details = program.stateMap(state.package).get(namespace) as Model | undefined; + const packageOptionsRaw = details?.properties.get("options")?.type as Model | undefined; + + const packageOptions = [...(packageOptionsRaw?.properties.entries() ?? [])] + .map(([k, { type }]) => { + // This condition is enforced by the definition of `dec package` + if (type.kind === "Boolean" || type.kind === "String" || type.kind === "Number") { + return [k, type.value] as [string, unknown]; + } else throw new Error(`Unexpected option type ${type.kind}`); + }) + .filter((v) => !!v) as [string, unknown][]; + + return { + package: ( + (details?.properties.get("name") as ModelProperty | undefined)?.type as + | StringLiteral + | undefined + )?.value, + + options: Object.fromEntries(packageOptions), + + imports: [...(importMap.get(namespace) ?? [])], + + declarations: declarationMap.get(namespace), + source: namespace, + } as ProtoFile; + }); + + checkForNamespaceCollisions(files); + + return files; + + /** + * Recursively searches a namespace for declarations that should be reified as Protobuf. + * + * @param namespace - the namespace to analyze + * @returns an array of declarations + */ + function addDeclarationsOfPackage(namespace: Namespace) { + const models = [...namespace.models.values()]; + + // Eagerly visit all models in the namespace. + for (const model of models) { + // Don't eagerly visit externs + if ( + // Don't eagerly visit externs + !program.stateMap(state.externRef).has(model) && + // Only eagerly visit models where every field has a field index annotation. + ([...model.properties.values()].every((p) => program.stateMap(state.fieldIndex).has(p)) || + // OR where the model has been explicitly marked as a message. + program.stateSet(state.message).has(model)) + ) { + visitModel(model, model); + } + } + + const interfacesInNamespace = new Set( + serviceInterfaces.filter((iface) => isDeclaredInNamespace(iface, namespace)) + ); + + // Each interface will be reified as a `service` declaration. + const declarations = declarationMap.get(namespace)!; + for (const iface of interfacesInNamespace) { + declarations.push({ + kind: "service", + name: iface.name, + // The service's methods are just projections of the interface operations. + operations: [...iface.operations.values()].map(toMethodFromOperation), + }); + } + } + + // #region inline helpers + + /** + * @param operation - the operation to convert + * @returns a corresponding method declaration + */ + function toMethodFromOperation(operation: Operation): ProtoMethodDeclaration { + const streamingMode = program.stateMap(state.stream).get(operation) ?? StreamingMode.None; + + return { + kind: "method", + stream: streamingMode, + name: capitalize(operation.name), + input: addImportSourceForProtoIfNeeded( + program, + addInputParams(operation.parameters, operation), + operation, + operation.parameters + ), + returns: addImportSourceForProtoIfNeeded( + program, + addReturnType(operation.returnType, operation), + operation, + operation.returnType as NamespaceTraversable + ), + }; + } + + /** + * Checks a parameter Model satisfies the constraints for a Protobuf method input and adds it to the declarations, + * returning a ProtoRef to the generated named message. + * + * @param model - the model to add + * @returns a reference to the model's message + */ + function addInputParams(paramsModel: Model, operation: Operation): ProtoRef { + const effectiveModel = computeEffectiveModel( + paramsModel, + capitalize(operation.name) + "Request" + ); + + /* c8 ignore start */ + + // Not sure if this can or can't actually happen at runtime, but we'll defensively handle it anyway. + if (!effectiveModel) { + reportDiagnostic(program, { + code: "unsupported-input-type", + messageId: "unconvertible", + target: paramsModel, + }); + + return unreachable("unsupported input type"); + } + /* c8 ignore stop */ + + return checkExtern(effectiveModel, operation); + } + + /** + * Returns an extern ref if the given type is an instance of `Extern`, otherwise returns a ref to the model's name. + */ + function checkExtern(model: Model, relativeSource: Model | Operation): ProtoRef { + const extern = program.stateMap(state.externRef).get(model) as [string, string] | undefined; + if (extern) { + typeWantsImport(program, relativeSource, extern[0]); + return ref(extern[1]); + } + + return ref(model.name); + } + + /** + * Gets a cached intrinsic type. This will also attach desired imports to the relative reference source. + */ + function getCachedExternType( + program: Program, + relativeSource: Operation | Model, + name: string + ): ProtoRef { + let cache = _protoExternMap.get(program); + + if (!cache) { + cache = new Map(); + _protoExternMap.set(program, cache); + } + + const cachedRef = cache.get(name); + + if (cachedRef) { + const [source, ref] = cachedRef; + typeWantsImport(program, relativeSource, source); + return ref; + } + + const [emptyType, diagnostics] = program.resolveTypeReference(name); + + if (!emptyType) { + throw new Error( + `Could not resolve the empty type: ${diagnostics.map(formatDiagnostic).join(EOL)}` + ); + } + + const extern = program.stateMap(state.externRef).get(emptyType) as [string, string] | undefined; + + if (!extern) { + throw new Error(`Unexpected: '${name}' was resolved but is not an extern type.`); + } + + const [source, protoName] = extern; + typeWantsImport(program, relativeSource, source); + const result = ref(protoName); + + cache.set(name, [source, result]); + + return result; + } + + /** + * Checks that a return type is a Model and converts it to a message, adding it to the declarations and returning + * a reference to its name. + * + * @param t - the model to add + * @param operationName - the name of the originating operation, used to compute a synthetic model name if required + * @returns a reference to the model's message + */ + function addReturnType(t: Type, operation: Operation): ProtoRef { + switch (t.kind) { + case "Model": + return addReturnModel(t, operation); + case "Intrinsic": + return addIntrinsicType(t, operation); + /* eslint-ignore-next-line no-fallthrough */ + default: + reportDiagnostic(program, { + code: "unsupported-return-type", + target: getOperationReturnSyntaxTarget(operation), + }); + + return unreachable("unsupported return type"); + } + } + + /** + * Adds an intrinsic type. Intrinsics are assumed to map to Extern types, so this will add the appropriate import. + * + * @param t - the intrinsic type to add + * @param relativeSource - the relative source of the type + * @returns a reference to the type's message + */ + function addIntrinsicType(t: IntrinsicType, relativeSource: Operation | Model): ProtoRef { + switch (t.name) { + case "unknown": + return getCachedExternType(program, relativeSource, "TypeSpec.Protobuf.WellKnown.Any"); + case "void": { + return getCachedExternType(program, relativeSource, "TypeSpec.Protobuf.WellKnown.Empty"); + } + } + + reportDiagnostic(program, { + code: "unsupported-intrinsic", + format: { type: t.name }, + target: t, + }); + + return unreachable("unsupported intrinsic type"); + } + + /** + * Converts a TypeSpec Model to a Protobuf Ref in return position, adding a corresponding message if necessary. + * + * @param m - the model to add to the Protofile. + * @returns a Protobuf reference to the model + */ + function addReturnModel(m: Model, operation: Operation): ProtoRef { + const extern = program.stateMap(state.externRef).get(m) as [string, string] | undefined; + if (extern) { + typeWantsImport(program, operation, extern[0]); + return ref(extern[1]); + } + + const effectiveModel = computeEffectiveModel(m, capitalize(operation.name) + "Response"); + if (effectiveModel) { + return ref(effectiveModel.name); + } + + reportDiagnostic(program, { + code: "unsupported-return-type", + target: getOperationReturnSyntaxTarget(operation), + }); + + return unreachable("unsupported return type"); + } + + /** + * Converts a TypeSpec type to a Protobuf type, adding a corresponding message if necessary. + * + * @param t - the type to add to the ProtoFile. + * @returns a Protobuf type corresponding to the given type + */ + function addType(t: Type, relativeSource: Model | Operation): ProtoType { + // Exit early if this type is an extern. + const extern = program.stateMap(state.externRef).get(t) as [string, string] | undefined; + if (extern) { + typeWantsImport(program, relativeSource, extern[0]); + return ref(extern[1]); + } + + if (isMap(program, t)) { + const mapType = mapToProto(t as Model, relativeSource); + mapImportSourceInformation.set(mapType, [relativeSource, t as NamespaceTraversable]); + return mapType; + } + + // Arrays transform into repeated fields, so we'll silently replace `t` with the array's member if this is an array. + // The `repeated` keyword will be added when the field is composed. + if (isArray(t)) { + return arrayToProto(t as Model, relativeSource); + } + + switch (t.kind) { + case "Model": + // If we came from another model and this model is anonymous, then we can't reference it by name. + if (t.name === "" && relativeSource.kind === "Model") { + reportDiagnostic(program, { + code: "anonymous-model", + target: t, + }); + return unreachable("anonymous model"); + } + visitModel(t, relativeSource); + return ref(t.name); + case "Enum": + visitEnum(t); + return ref(t.name); + case "Scalar": + return scalarToProto(t); + case "Intrinsic": + return addIntrinsicType(t, relativeSource); + default: + reportDiagnostic(program, { + code: "unsupported-field-type", + messageId: "unconvertible", + format: { + type: t.kind, + }, + target: t, + }); + return unreachable("unsupported field type"); + } + } + + function mapToProto(t: Model, relativeSource: Model | Operation): ProtoMap { + const [keyType, valueType] = t.templateMapper!.args; + + // A map's value cannot be another map. + if (isMap(program, valueType)) { + reportDiagnostic(program, { + code: "unsupported-field-type", + messageId: "recursive-map", + target: valueType, + }); + return unreachable("recursive map"); + } + + // This is a core compile error. + if (!keyType || !valueType) return unreachable("nonexistent map key or value type"); + + // Key constraint (integral | string) is enforced by the type constraint on the `Map<>` type. + const keyProto = addType(keyType, relativeSource); + const valueProto = addType(valueType, relativeSource) as ProtoRef | ProtoScalar; + + return map(keyProto[1] as "string" | ScalarIntegralName, valueProto); + } + + function arrayToProto(t: Model, relativeSource: Model | Operation): ProtoType { + const valueType = (t as Model).templateMapper!.args[0]; + + // Nested arrays are not supported. + if (isArray(valueType)) { + reportDiagnostic(program, { + code: "nested-array", + target: valueType, + }); + return ref(""); + } + + return addType(valueType, relativeSource); + } + + function getProtoScalarsMap(program: Program): Map { + // The type references are different object identities in different programs, so we need to cache the map per program. + // This really only affects tests in our current use case, but someone could be using the compiler API to compile + // multiple programs and it also affects that. + let scalarMap; + if (_protoScalarsMap.has(program)) { + scalarMap = _protoScalarsMap.get(program)!; + } else { + const entries = [ + [program.resolveTypeReference("TypeSpec.bytes"), scalar("bytes")], + [program.resolveTypeReference("TypeSpec.boolean"), scalar("bool")], + [program.resolveTypeReference("TypeSpec.string"), scalar("string")], + [program.resolveTypeReference("TypeSpec.int32"), scalar("int32")], + [program.resolveTypeReference("TypeSpec.int64"), scalar("int64")], + [program.resolveTypeReference("TypeSpec.uint32"), scalar("uint32")], + [program.resolveTypeReference("TypeSpec.uint64"), scalar("uint64")], + [program.resolveTypeReference("TypeSpec.float32"), scalar("float")], + [program.resolveTypeReference("TypeSpec.float64"), scalar("double")], + [program.resolveTypeReference("TypeSpec.Protobuf.sfixed32"), scalar("sfixed32")], + [program.resolveTypeReference("TypeSpec.Protobuf.sfixed64"), scalar("sfixed64")], + [program.resolveTypeReference("TypeSpec.Protobuf.sint32"), scalar("sint32")], + [program.resolveTypeReference("TypeSpec.Protobuf.sint64"), scalar("sint64")], + [program.resolveTypeReference("TypeSpec.Protobuf.fixed32"), scalar("fixed32")], + [program.resolveTypeReference("TypeSpec.Protobuf.fixed64"), scalar("fixed64")], + ] as const; + + for (const [[type, diagnostics]] of entries) { + if (!type) { + const diagnosticString = diagnostics.map(formatDiagnostic).join(EOL); + throw new Error( + `Failed to construct TypeSpec -> Protobuf scalar map. Unexpected failure to resolve TypeSpec scalar: ${diagnosticString}` + ); + } + } + + scalarMap = new Map(entries.map(([[type], scalar]) => [type!, scalar])); + + _protoScalarsMap.set(program, scalarMap); + } + // Lazy initialize this map of known proto scalars. + + return scalarMap; + } + + function scalarToProto(t: Scalar): ProtoType { + const fullName = getTypeName(t); + + const protoType = getProtoScalarsMap(program).get(t); + + if (!protoType) { + if (t.baseScalar) { + return scalarToProto(t.baseScalar); + } else { + reportDiagnostic(program, { + code: "unsupported-field-type", + messageId: "unknown-scalar", + format: { + name: fullName, + }, + target: t, + }); + return unreachable("unknown scalar"); + } + } + + return protoType; + } + + function computeEffectiveModel(model: Model, anonymousModelName: string): Model | undefined { + if (effectiveModelCache.has(model)) return effectiveModelCache.get(model); + + let effectiveModel = getEffectiveModelType(program, model); + + if (effectiveModel.name === "") { + // Name the model automatically if it is anonymous + effectiveModel = program.checker.createAndFinishType({ + ...model, + name: anonymousModelName, + }); + } + + if (!program.stateMap(state.externRef).has(effectiveModel)) { + visitModel(effectiveModel, model); + } + + effectiveModelCache.set(model, effectiveModel); + + return effectiveModel; + } + // #endregion + + function checkForNamespaceCollisions(files: ProtoFile[]) { + const namespaces = new Set(); + + for (const file of files) { + if (namespaces.has(file.package)) { + reportDiagnostic(program, { + code: "namespace-collision", + format: { + name: `"${file.package}"` ?? "", + }, + target: file.source, + }); + } + + namespaces.add(file.package); + } + } + + /** + * @param model - the Model to convert + * @returns a corresponding message declaration + */ + function toMessage(model: Model): ProtoMessageDeclaration { + return { + kind: "message", + name: model.name, + reservations: program.stateMap(state.reserve).get(model), + declarations: [...model.properties.values()].map((f) => toMessageBodyDeclaration(f, model)), + }; + } + + /** + * @param property - the ModelProperty to convert + * @returns a corresponding declaration + */ + function toMessageBodyDeclaration( + property: ModelProperty, + model: Model + ): ProtoMessageBodyDeclaration { + if (property.type.kind === "Union") { + // Unions are difficult to represent in protobuf, so for now we don't support them. + // See : https://github.com/microsoft/typespec/issues/1854 + reportDiagnostic(program, { + code: "unsupported-field-type", + messageId: "union", + target: property, + }); + return unreachable("union"); + } + + const fieldIndex = program.stateMap(state.fieldIndex).get(property) as number | undefined; + const fieldIndexNode = property.decorators.find((d) => d.decorator === $field)?.args[0].node; + + if (fieldIndex === undefined) { + reportDiagnostic(program, { + code: "field-index", + messageId: "missing", + format: { + name: property.name, + }, + target: property, + }); + } + + if (fieldIndex && !fieldIndexNode) + throw new Error("Failed to recover field decorator argument."); + + const reservations = program.stateMap(state.reserve).get(model) as Reservation[] | undefined; + + if (reservations) { + for (const reservation of reservations) { + if (typeof reservation === "string" && reservation === property.name) { + reportDiagnostic(program, { + code: "field-name", + messageId: "user-reserved", + format: { + name: property.name, + }, + target: getPropertyNameSyntaxTarget(property), + }); + } else if ( + fieldIndex !== undefined && + typeof reservation === "number" && + reservation === fieldIndex + ) { + reportDiagnostic(program, { + code: "field-index", + messageId: "user-reserved", + format: { + index: fieldIndex.toString(), + }, + // Fail over to using the model if the field index node is missing... this should never occur but it's the + // simplest way to satisfy the type system. + target: fieldIndexNode ?? model, + }); + } else if ( + fieldIndex !== undefined && + Array.isArray(reservation) && + fieldIndex >= reservation[0] && + fieldIndex <= reservation[1] + ) { + reportDiagnostic(program, { + code: "field-index", + messageId: "user-reserved-range", + format: { + index: fieldIndex.toString(), + }, + target: fieldIndexNode ?? model, + }); + } + } + } + + const field: ProtoFieldDeclaration = { + kind: "field", + name: property.name, + type: addImportSourceForProtoIfNeeded( + program, + addType(property.type, model), + model, + property.type as NamespaceTraversable + ), + index: program.stateMap(state.fieldIndex).get(property), + }; + + // Determine if the property type is an array + if (isArray(property.type)) field.repeated = true; + + return field; + } + + /** + * @param e - the Enum to convert + * @returns a corresponding protobuf enum declaration + * + * INVARIANT: the enum's members must be integer values + */ + function toEnum(e: Enum): ProtoEnumDeclaration { + const needsAlias = new Set([...e.members.values()].map((v) => v.value)).size !== e.members.size; + + return { + kind: "enum", + name: e.name, + allowAlias: needsAlias, + variants: [...e.members.values()].map(({ name, value }) => [name, value as number]), + }; + } + + type NamespaceTraversable = + | Enum + | Model + | Interface + | Union + | Operation + | Namespace + | IntrinsicType; + + function getPackageOfType(program: Program, t: NamespaceTraversable): Namespace | null { + /* c8 ignore start */ + + // Most of this should be unreachable, but we'll guard it with diagnostics anyway in case of eventual synthetic types. + + switch (t.kind) { + case "Intrinsic": + // Intrinsics are all handled explicitly. + return null; + case "Enum": + case "Model": + case "Union": + case "Interface": + if (!t.namespace) { + return null; + } else { + return getPackageOfType(program, t.namespace); + } + case "Operation": { + const logicalParent = t.interface ?? t.namespace; + if (!logicalParent) { + return null; + } else { + return getPackageOfType(program, logicalParent); + } + } + case "Namespace": + if (packages.has(t)) return t; + + if (!t.namespace) { + return null; + } else { + return getPackageOfType(program, t.namespace); + } + } + /* c8 ignore stop */ + } + + function addImportSourceForProtoIfNeeded( + program: Program, + pt: T, + dependent: Model | Operation, + dependency: NamespaceTraversable + ): T { + { + // Early escape for intrinsics + if (dependency.kind === "Intrinsic") { + // Intrinsics and imports are handled explicitly by the emitter. + return pt; + } + } + + { + // Early escape for externs + let effectiveModel: Model | undefined; + if ( + program.stateMap(state.externRef).has(dependency) || + (dependency.kind === "Model" && + (effectiveModel = effectiveModelCache.get(dependency)) && + program.stateMap(state.externRef).has(effectiveModel)) + ) { + return pt; + } + } + + if (isArray(dependency)) { + return addImportSourceForProtoIfNeeded( + program, + pt, + dependent, + (dependency as Model).templateMapper!.args[0] as NamespaceTraversable + ); + } + try { + // If we had an error producing an "unreachable" type, we would actually reach it during validation below, so the + // try/catch allows us to pass the unreachable back up the chain. + return matchType(pt, { + map(k, v) { + const mapInfo = mapImportSourceInformation.get(pt as ProtoMap); + return mapInfo !== undefined + ? (map( + k, + addImportSourceForProtoIfNeeded(program, v, mapInfo[0], mapInfo[1]) as + | ProtoRef + | ProtoScalar + // Anything else is unreachable by construction. + ) as T) + : pt; + }, + scalar() { + return pt; + }, + ref(r) { + const [dependentPackage, dependencyPackage] = [ + getPackageOfType(program, dependent), + getPackageOfType(program, dependency), + ]; + + if ( + dependentPackage === null || + dependencyPackage === null || + dependentPackage === dependencyPackage + ) + return pt; + + const dependencyDetails = program.stateMap(state.package).get(dependencyPackage) as + | Model + | undefined; + + const dependencyPackageName = ( + dependencyDetails?.properties.get("name")?.type as StringLiteral | undefined + )?.value; + + const dependencyPackagePrefix = + dependencyPackageName === undefined || dependencyPackageName === "" + ? "" + : dependencyPackageName + "."; + + const dependencyFileName = + (dependencyPackageName?.split(".") ?? ["main"]).join("/") + ".proto"; + + importMap.get(dependentPackage)?.add(dependencyFileName); + + return ref(dependencyPackagePrefix + r) as T; + }, + }); + } catch { + return pt; + } + } +} + +function isArray(t: Type) { + return t.kind === "Model" && t.name === "Array" && t.namespace?.name === "TypeSpec"; +} + +/** + * Simple utility function to capitalize a string. + */ +function capitalize(s: S) { + return (s.slice(0, 1).toUpperCase() + s.slice(1)) as Capitalize; +} + +/** + * Gets the syntactic return type target for an operation. + * + * Helps us squiggle the right things for operation return types. + * + * See https://github.com/microsoft/typespec/issues/1650. This issue tracks helpers for doing this without requiring + * emitters to implement this functionality. + */ +function getOperationReturnSyntaxTarget(op: Operation): DiagnosticTarget { + const signature = op.node.signature; + switch (signature.kind) { + case SyntaxKind.OperationSignatureDeclaration: + return signature.returnType; + case SyntaxKind.OperationSignatureReference: + return op; + default: + const __exhaust: never = signature; + throw new Error( + `Internal Emitter Error: reached unreachable operation signature: ${op.node.signature.kind}` + ); + } +} + +/** + * Gets the syntactic position of a model property name. + * + * See https://github.com/microsoft/typespec/issues/1650. This issue tracks helpers for doing this without requiring + * emitters to implement this functionality. + */ +function getPropertyNameSyntaxTarget(property: ModelProperty): DiagnosticTarget { + const node = property.node; + + switch (node.kind) { + case SyntaxKind.ModelProperty: + return node.id; + case SyntaxKind.ModelSpreadProperty: + return node; + case SyntaxKind.ProjectionModelProperty: + case SyntaxKind.ProjectionModelSpreadProperty: + return property; + default: + const __exhaust: never = node; + throw new Error( + `Internal Emitter Error: reached unreachable model property node: ${property.node.kind}` + ); + } +} diff --git a/packages/protobuf/src/write.ts b/packages/protobuf/src/write.ts new file mode 100644 index 00000000000..5826d5d4048 --- /dev/null +++ b/packages/protobuf/src/write.ts @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + matchType, + ProtoDeclaration, + ProtoEnumDeclaration, + ProtoFieldDeclaration, + ProtoFile, + ProtoMessageDeclaration, + ProtoMethodDeclaration, + ProtoOneOfDeclaration, + ProtoServiceDeclaration, + ProtoType, + StreamingMode, +} from "./ast.js"; + +// This module defines how to emit the text representation of a ProtoFile AST. + +/** + * Header for the top of all emitted proto files. + * + * We only support Protobuf 3 syntax. + */ +export const PROTO_HEADER = `/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; +`; + +/** + * Write the given `file` to a string. + */ +export function writeProtoFile(file: ProtoFile): string { + let result = PROTO_HEADER; + + if (file.package) result += `\npackage ${file.package};\n`; + + for (const _import of file.imports) { + result += `\nimport "${_import}";`; + } + + if (file.imports.length > 0) result += "\n"; + + const opts = Object.entries(file.options); + for (const [name, valueData] of opts) { + const value = typeof valueData === "string" ? `"${valueData}"` : valueData.toString(); + result += `\noption ${name} = ${value};`; + } + + // Give the declarations a little breathing room if options were provided + if (opts.length > 0) result += "\n"; + + for (const decl of file.declarations) { + result += "\n" + collect(writeDeclaration(decl)).join("\n") + "\n"; + } + + return result; +} + +/** + * Write the given `decl` to a line iterable. + */ +function* writeDeclaration(decl: ProtoDeclaration): Iterable { + switch (decl.kind) { + case "message": + yield* writeMessage(decl); + return; + case "service": + yield* writeService(decl); + return; + case "field": + yield writeField(decl); + return; + case "oneof": + yield* writeOneOf(decl); + return; + case "enum": + yield* writeEnum(decl); + return; + case "method": + yield writeMethod(decl); + return; + /* c8 ignore next 5 */ + default: + const __exhaust: never = decl; + throw __exhaust; + } +} + +/** + * Write the given message `decl` to a line iterable. + */ +function* writeMessage(decl: ProtoMessageDeclaration): Iterable { + const head = `message ${decl.name} {`; + const tail = "}"; + + if (decl.declarations.length > 0 || decl.reservations?.length) { + yield head; + yield* indent(writeReservations(decl)); + yield* indent(flatMap(decl.declarations, writeDeclaration)); + yield tail; + } else yield head + tail; +} + +function* writeReservations(decl: ProtoMessageDeclaration): Iterable { + const { reservedNumbers, reservedNames } = selectMap( + decl.reservations ?? [], + (v) => (typeof v === "number" || Array.isArray(v) ? "reservedNumbers" : "reservedNames"), + { + reservedNumbers: (v) => (Array.isArray(v) ? v[0] + " to " + v[1] : v.toString()), + reservedNames: (v) => `"${v.toString()}"`, + } + ); + + if (reservedNumbers.length + reservedNames.length > 0) { + if (reservedNumbers.length > 0) yield `reserved ${reservedNumbers.join(", ")};`; + if (reservedNames.length > 0) yield `reserved ${reservedNames.join(", ")};`; + yield ""; + } +} + +function* writeService(decl: ProtoServiceDeclaration): Iterable { + const head = `service ${decl.name} {`; + const tail = "}"; + + if (decl.operations.length > 0) { + yield head; + yield* indent(flatMap(decl.operations, writeDeclaration)); + yield tail; + } else yield head + tail; +} + +function writeMethod(decl: ProtoMethodDeclaration): string { + const [inStream, outStream] = [ + decl.stream & StreamingMode.In, + decl.stream & StreamingMode.Out, + ].map((v) => (v ? "stream " : "")); + + return `rpc ${decl.name}(${inStream}${writeType(decl.input)}) returns (${outStream}${writeType( + decl.returns + )});`; +} + +function* writeOneOf(decl: ProtoOneOfDeclaration): Iterable { + // OneOf declarations must have at least one element, so no need to check for declarations + yield `oneof ${decl.name} {`; + yield* indent(flatMap(decl.declarations, writeDeclaration)); + yield "}"; +} + +function* writeEnum(decl: ProtoEnumDeclaration): Iterable { + yield `enum ${decl.name} {`; + if (decl.allowAlias) { + yield " option allow_alias = true;"; + + if (decl.variants.length > 0) yield ""; + } + yield* indent(flatMap(decl.variants, ([name, idx]) => `${name} = ${idx};`)); + yield "}"; +} + +function writeField(decl: ProtoFieldDeclaration): string { + const prefix = decl.repeated ? "repeated " : ""; + return prefix + `${writeType(decl.type)} ${decl.name} = ${decl.index};`; +} + +function writeType(type: ProtoType): string { + return matchType(type, { + map: (k, v) => `map<${k}, ${writeType(v)}>`, + ref: (r) => r, + scalar: (s) => s, + }); +} + +// #region utils + +/** + * Indents an iterable of strings by prepending an amount of spaces to each item + * in the iterable. + * + * @param it - the string iterable to indent + * @param depth - the indentation depth in spaces, defaults to 2 + */ +function* indent(it: Iterable, depth: number = 2): Iterable { + for (const value of it) { + if (value !== "") { + yield " ".repeat(depth) + value; + } else yield value; + } +} + +/** + * A version of flatMap that works with generic iterables. + * + * @param it - the iterable to flatten and map + * @param f - the function to run on the items of `it` + */ +function* flatMap(it: Iterable, f: (v: T1) => T2 | Iterable): Iterable { + for (const value of it) { + const result = f(value); + if (typeof result === "object" && result !== null && Symbol.iterator in result) { + yield* result as Iterable; + } else { + yield result as T2; + } + } +} + +/** + * Collects an iterable into an array. Having this as a callable function is useful for writing functional combinations. + * + * @param it - the iterable to collect + * @returns an array with all the items in the iterable + */ +function collect(it: Iterable): T[] { + return [...it]; +} + +/** + * A helper function that allows categorizing items from an iterable into groups and running a different map function + * for each group. + * + * @param source - the iterable to apply the selection and mapping to + * @param select - a function that is applied to each item and produces a selector + * @param delegates - a record of selectors to mapping functions + * @returns a record of selectors to arrays of results produced by each delegate + */ +function selectMap unknown }>( + source: Iterable, + select: (v: TIn) => keyof Delegates, + delegates: Delegates +) { + const result = Object.fromEntries(Object.keys(delegates).map((k) => [k, []])) as unknown as { + [K in keyof Delegates]: ReturnType[]; + }; + for (const value of source) { + const k = select(value); + result[k].push(delegates[k](value) as any); + } + + return result; +} + +// #endregion diff --git a/packages/protobuf/test/include/foo/bar.proto b/packages/protobuf/test/include/foo/bar.proto new file mode 100644 index 00000000000..ac62127a60f --- /dev/null +++ b/packages/protobuf/test/include/foo/bar.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Bar { + int32 field = 1; +} diff --git a/packages/protobuf/test/scenarios.spec.ts b/packages/protobuf/test/scenarios.spec.ts new file mode 100644 index 00000000000..4b4e2bb659b --- /dev/null +++ b/packages/protobuf/test/scenarios.spec.ts @@ -0,0 +1,226 @@ +import assert from "assert"; +import path from "path"; +import url from "url"; + +import micromatch from "micromatch"; + +import { formatDiagnostic } from "@typespec/compiler"; +import { + createTestHost, + resolveVirtualPath, + TypeSpecTestLibrary, +} from "@typespec/compiler/testing"; +import { readdirSync, statSync } from "fs"; +import { mkdir, readdir, readFile, rm, stat, writeFile } from "fs/promises"; + +const SCENARIOS_DIRECTORY = url.fileURLToPath(new url.URL("../../test/scenarios", import.meta.url)); + +const shouldRecord = process.env.RECORD === "true"; +const patternsToRun = process.env.RUN_SCENARIOS?.split(",") ?? ["*"]; + +const TypeSpecProtobufTestLibrary: TypeSpecTestLibrary = { + name: "@typespec/protobuf", + packageRoot: path.resolve(url.fileURLToPath(import.meta.url), "../../../"), + files: [ + { realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typespec/protobuf" }, + { + realDir: "dist/src", + pattern: "*.js", + virtualPath: "./node_modules/@typespec/protobuf/dist/src", + }, + { realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typespec/protobuf/lib" }, + ], +}; + +describe("protobuf scenarios", function () { + const scenarios = readdirSync(SCENARIOS_DIRECTORY) + .map((dn) => path.join(SCENARIOS_DIRECTORY, dn)) + .filter((dn) => statSync(dn).isDirectory()); + + for (const scenario of scenarios) { + const scenarioName = path.basename(scenario); + + const shouldRun = micromatch.isMatch(scenarioName, patternsToRun); + + shouldRun && + it(scenarioName, async function () { + const inputFiles = await readdirRecursive(path.join(scenario, "input")); + const emitResult = await doEmit(inputFiles); + + const expectationDirectory = path.resolve(scenario, "output"); + const diagnosticsExpectationPath = path.resolve(scenario, "diagnostics.txt"); + + if (shouldRecord) { + // Write new output to the scenario's output folder. + + await writeExpectationDirectory(expectationDirectory, emitResult.files); + + await rm(diagnosticsExpectationPath, { force: true }); + + if (emitResult.diagnostics.length > 0) { + const diagnostics = emitResult.diagnostics.join("\n"); + + await writeFile(diagnosticsExpectationPath, diagnostics); + } + } else { + // It's an error if any file in the expected files is missing, if any file in the output files doesn't have a + // corresponding expectation, or if any file in the output files doesn't match its corresponding output file + // character for character. + + let err: Error | undefined = undefined; + + // `throwIfNoEntry` is not supported with promisified fs.promises.stat. + if (!statSync(expectationDirectory, { throwIfNoEntry: false })) { + assert.strictEqual( + Object.entries(emitResult.files).length, + 0, + "no expectations exist, but output files were generated" + ); + } else { + const expectedFiles = await readdirRecursive(expectationDirectory); + + // Need to defer this error until we've checked for diagnostics below. If diagnostics were unexpectedly + // raised and inhibited emit, that should be the primary error, not this one. + try { + assertFilesAsExpected(emitResult.files, expectedFiles); + } catch (e: unknown) { + err = e as Error; + } + } + + let expectedDiagnostics: string; + try { + expectedDiagnostics = (await readFile(diagnosticsExpectationPath)).toString("utf-8"); + } catch { + expectedDiagnostics = ""; + } + + // Fix the start of lines on Windows + const processedDiagnostics = + process.platform === "win32" + ? emitResult.diagnostics.map((d) => d.replace(/^Z:/, "")) + : emitResult.diagnostics; + + const diagnostics = processedDiagnostics.join("\n"); + + assert.strictEqual(diagnostics, expectedDiagnostics, "expected equivalent diagnostics"); + + if (err) throw err; + } + }); + } +}); + +interface EmitResult { + files: Record; + diagnostics: string[]; +} + +async function doEmit(files: Record): Promise { + const baseOutputPath = resolveVirtualPath("test-output/"); + + const host = await createTestHost({ + libraries: [TypeSpecProtobufTestLibrary], + }); + + for (const [fileName, content] of Object.entries(files)) { + host.addTypeSpecFile(fileName, content); + } + + const [, diagnostics] = await host.compileAndDiagnose("main.tsp", { + outputDir: baseOutputPath, + noEmit: false, + emitters: { + "@typespec/protobuf": {}, + }, + }); + + return { + files: Object.fromEntries( + [...host.fs.entries()] + .filter(([name]) => name.startsWith(baseOutputPath)) + .map(([name, value]) => [name.replace(baseOutputPath, ""), value]) + ), + diagnostics: diagnostics.map(formatDiagnostic), + }; +} + +function assertFilesAsExpected( + outputFiles: Record, + expectedFiles: Record +) { + for (const fn of Object.keys(expectedFiles)) { + assert.ok( + Object.prototype.hasOwnProperty.call(outputFiles, fn), + `expected file ${fn} was not produced` + ); + } + + for (const [fn, content] of Object.entries(outputFiles)) { + const expectedContent = expectedFiles[fn]; + + assert.ok(expectedContent, `output file ${fn} has no corresponding expectation`); + + assert.strictEqual(content, expectedContent); + } +} + +/** + * Writes an expectation map to disk. + * + * @param expectationDirectory - The directory to write to. + * @param outputFiles - A map of relative paths to file contents. + */ +async function writeExpectationDirectory( + expectationDirectory: string, + outputFiles: Record +) { + const fileEntries = Object.entries(outputFiles); + + // It'll be annoying to fiddle with .gitkeep files, so let's omit the `output` directory if it's empty. + if (fileEntries.length === 0) { + return; + } + + await rm(expectationDirectory, { recursive: true, force: true }); + + await mkdir(expectationDirectory); + + for (const [fn, content] of fileEntries) { + const fullPath = path.join(expectationDirectory, fn); + await mkdir(path.dirname(fullPath), { recursive: true }); + await writeFile(fullPath, content); + } +} + +/** + * @param dir - The directory to read recursively. + * @param base - The base directory to use for relative paths. + * @returns A map of relative paths to file contents. + */ +async function readdirRecursive(dir: string, base: string = dir): Promise> { + const res: Record = {}; + + for (const entry of (await readdir(dir)).map((e) => path.join(dir, e))) { + const stats = await stat(entry); + + if (stats.isDirectory()) { + for (const [name, content] of Object.entries(await readdirRecursive(entry, base))) { + res[name] = content; + } + } else if (stats.isFile()) { + const content = (await readFile(entry)).toString("utf-8"); + + const relativePath = path.relative(base, entry); + + const correctedPath = + process.platform === "win32" ? relativePath.replace(/\\/g, "/") : relativePath; + + res[correctedPath] = content; + } else { + throw new Error("Unsupported file type."); + } + } + + return res; +} diff --git a/packages/protobuf/test/scenarios/addressbook/input/addressbook.tsp b/packages/protobuf/test/scenarios/addressbook/input/addressbook.tsp new file mode 100644 index 00000000000..d0fdd62b034 --- /dev/null +++ b/packages/protobuf/test/scenarios/addressbook/input/addressbook.tsp @@ -0,0 +1,27 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "addressbook", +}) +namespace AddressBook; + +enum PhoneType { + MOBILE: 0, + HOME: 1, + WORK: 2, +} + +model PhoneNumber { + @field(1) number: string; + @field(2) type: PhoneType; +} + +model Person { + @field(1) name: string; + @field(2) id: int32; + @field(3) email: string; + @field(4) phones: PhoneNumber[]; + @field(5) last_updated: WellKnown.Timestamp; +} diff --git a/packages/protobuf/test/scenarios/addressbook/input/main.tsp b/packages/protobuf/test/scenarios/addressbook/input/main.tsp new file mode 100644 index 00000000000..4153b8ec33c --- /dev/null +++ b/packages/protobuf/test/scenarios/addressbook/input/main.tsp @@ -0,0 +1,13 @@ +import "@typespec/protobuf"; +import "./addressbook.tsp"; + +using TypeSpec.Protobuf; +using TypeSpec.Protobuf.WellKnown; + +@package +namespace Example; + +@Protobuf.service +interface AddressBookService { + addPerson(@field(1) person: AddressBook.Person): Empty; +} diff --git a/packages/protobuf/test/scenarios/addressbook/output/@typespec/protobuf/addressbook.proto b/packages/protobuf/test/scenarios/addressbook/output/@typespec/protobuf/addressbook.proto new file mode 100644 index 00000000000..2fdbab6edee --- /dev/null +++ b/packages/protobuf/test/scenarios/addressbook/output/@typespec/protobuf/addressbook.proto @@ -0,0 +1,26 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package addressbook; + +import "google/protobuf/timestamp.proto"; + +enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; +} + +message PhoneNumber { + string number = 1; + PhoneType type = 2; +} + +message Person { + string name = 1; + int32 id = 2; + string email = 3; + repeated PhoneNumber phones = 4; + google.protobuf.Timestamp last_updated = 5; +} diff --git a/packages/protobuf/test/scenarios/addressbook/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/addressbook/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..dde2c147648 --- /dev/null +++ b/packages/protobuf/test/scenarios/addressbook/output/@typespec/protobuf/main.proto @@ -0,0 +1,14 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +import "addressbook.proto"; +import "google/protobuf/empty.proto"; + +message AddPersonRequest { + addressbook.Person person = 1; +} + +service AddressBookService { + rpc AddPerson(AddPersonRequest) returns (google.protobuf.Empty); +} diff --git a/packages/protobuf/test/scenarios/anonymous-model/diagnostics.txt b/packages/protobuf/test/scenarios/anonymous-model/diagnostics.txt new file mode 100644 index 00000000000..1775d2f146a --- /dev/null +++ b/packages/protobuf/test/scenarios/anonymous-model/diagnostics.txt @@ -0,0 +1 @@ +/test/main.tsp:16:29 - error @typespec/protobuf/anonymous-model: anonymous models cannot be used in Protobuf messages \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/anonymous-model/input/main.tsp b/packages/protobuf/test/scenarios/anonymous-model/input/main.tsp new file mode 100644 index 00000000000..ef50660d206 --- /dev/null +++ b/packages/protobuf/test/scenarios/anonymous-model/input/main.tsp @@ -0,0 +1,24 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: { + @field(1) testNestedField: int32; + }; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/anonymous-package/input/main.tsp b/packages/protobuf/test/scenarios/anonymous-package/input/main.tsp new file mode 100644 index 00000000000..4c07fd10d79 --- /dev/null +++ b/packages/protobuf/test/scenarios/anonymous-package/input/main.tsp @@ -0,0 +1,19 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: int32; +} diff --git a/packages/protobuf/test/scenarios/anonymous-package/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/anonymous-package/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..b990bd3bf45 --- /dev/null +++ b/packages/protobuf/test/scenarios/anonymous-package/output/@typespec/protobuf/main.proto @@ -0,0 +1,15 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +message Input { + string testInputField = 1; +} + +message Output { + int32 testOutputField = 1; +} + +service Service { + rpc Foo(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/array-nested/diagnostics.txt b/packages/protobuf/test/scenarios/array-nested/diagnostics.txt new file mode 100644 index 00000000000..0c41dca15b1 --- /dev/null +++ b/packages/protobuf/test/scenarios/array-nested/diagnostics.txt @@ -0,0 +1 @@ +/test/.tsp/lib/lib.tsp:121:1 - error @typespec/protobuf/nested-array: nested arrays are not supported by the Protobuf emitter \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/array-nested/input/main.tsp b/packages/protobuf/test/scenarios/array-nested/input/main.tsp new file mode 100644 index 00000000000..64ad31180c3 --- /dev/null +++ b/packages/protobuf/test/scenarios/array-nested/input/main.tsp @@ -0,0 +1,20 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace com.azure.Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string[][]; +} + +model Output { + @field(1) testOutputField: int32[]; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/array/input/main.tsp b/packages/protobuf/test/scenarios/array/input/main.tsp new file mode 100644 index 00000000000..4fb4ba7ddbf --- /dev/null +++ b/packages/protobuf/test/scenarios/array/input/main.tsp @@ -0,0 +1,22 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.test", +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string[]; +} + +model Output { + @field(1) testOutputField: int32[]; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/array/output/@typespec/protobuf/com/azure/test.proto b/packages/protobuf/test/scenarios/array/output/@typespec/protobuf/com/azure/test.proto new file mode 100644 index 00000000000..c3bd262fb23 --- /dev/null +++ b/packages/protobuf/test/scenarios/array/output/@typespec/protobuf/com/azure/test.proto @@ -0,0 +1,18 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.test; + +message Input { + repeated string testInputField = 1; +} + +message Output { + repeated int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Foo(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/cross package references/input/main.tsp b/packages/protobuf/test/scenarios/cross package references/input/main.tsp new file mode 100644 index 00000000000..81470b434e1 --- /dev/null +++ b/packages/protobuf/test/scenarios/cross package references/input/main.tsp @@ -0,0 +1,27 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "A", +}) +namespace A { + model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; + } +} + +@package({ + name: "B", +}) +namespace B { + @Protobuf.service + interface Service { + foo(...Input): A.Output; + } + + model Input { + @field(1) testInputField: string; + } +} diff --git a/packages/protobuf/test/scenarios/cross package references/output/@typespec/protobuf/A.proto b/packages/protobuf/test/scenarios/cross package references/output/@typespec/protobuf/A.proto new file mode 100644 index 00000000000..0eae330f106 --- /dev/null +++ b/packages/protobuf/test/scenarios/cross package references/output/@typespec/protobuf/A.proto @@ -0,0 +1,10 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package A; + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} diff --git a/packages/protobuf/test/scenarios/cross package references/output/@typespec/protobuf/B.proto b/packages/protobuf/test/scenarios/cross package references/output/@typespec/protobuf/B.proto new file mode 100644 index 00000000000..bcaece25e2f --- /dev/null +++ b/packages/protobuf/test/scenarios/cross package references/output/@typespec/protobuf/B.proto @@ -0,0 +1,15 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package B; + +import "A.proto"; + +message Input { + string testInputField = 1; +} + +service Service { + rpc Foo(Input) returns (A.Output); +} diff --git a/packages/protobuf/test/scenarios/derived-scalar/input/main.tsp b/packages/protobuf/test/scenarios/derived-scalar/input/main.tsp new file mode 100644 index 00000000000..f02f998755f --- /dev/null +++ b/packages/protobuf/test/scenarios/derived-scalar/input/main.tsp @@ -0,0 +1,24 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", +}) +namespace Test; + +scalar MyInt32 extends int32; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: MyInt32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/derived-scalar/output/@typespec/protobuf/com/azure/Test.proto b/packages/protobuf/test/scenarios/derived-scalar/output/@typespec/protobuf/com/azure/Test.proto new file mode 100644 index 00000000000..32114a3b14a --- /dev/null +++ b/packages/protobuf/test/scenarios/derived-scalar/output/@typespec/protobuf/com/azure/Test.proto @@ -0,0 +1,18 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.Test; + +message Input { + string testInputField = 1; +} + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Foo(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/enum-nonintegral/diagnostics.txt b/packages/protobuf/test/scenarios/enum-nonintegral/diagnostics.txt new file mode 100644 index 00000000000..b021264163c --- /dev/null +++ b/packages/protobuf/test/scenarios/enum-nonintegral/diagnostics.txt @@ -0,0 +1,4 @@ +/test/main.tsp:18:1 - error @typespec/protobuf/unconvertible-enum: enums must explicitly assign exactly one integer to each member to be used in a Protobuf message +/test/main.tsp:19:3 - error @typespec/protobuf/unconvertible-enum: the first variant of an enum must be set to zero to be used in a Protobuf message +/test/main.tsp:23:1 - error @typespec/protobuf/unconvertible-enum: enums must explicitly assign exactly one integer to each member to be used in a Protobuf message +/test/main.tsp:24:3 - error @typespec/protobuf/unconvertible-enum: the first variant of an enum must be set to zero to be used in a Protobuf message \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/enum-nonintegral/input/main.tsp b/packages/protobuf/test/scenarios/enum-nonintegral/input/main.tsp new file mode 100644 index 00000000000..231edf79ea3 --- /dev/null +++ b/packages/protobuf/test/scenarios/enum-nonintegral/input/main.tsp @@ -0,0 +1,31 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; + @field(2) type: InputType; +} + +enum InputType { + FOO: 1, + BAR: "test", +} + +enum OutputType { + FOO, + BAR, +} + +model Output { + @field(1) testOutputField: OutputType; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/enum-nozero/diagnostics.txt b/packages/protobuf/test/scenarios/enum-nozero/diagnostics.txt new file mode 100644 index 00000000000..0ccdc9734ef --- /dev/null +++ b/packages/protobuf/test/scenarios/enum-nozero/diagnostics.txt @@ -0,0 +1 @@ +/test/main.tsp:19:3 - error @typespec/protobuf/unconvertible-enum: the first variant of an enum must be set to zero to be used in a Protobuf message \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/enum-nozero/input/main.tsp b/packages/protobuf/test/scenarios/enum-nozero/input/main.tsp new file mode 100644 index 00000000000..12b298f1b0f --- /dev/null +++ b/packages/protobuf/test/scenarios/enum-nozero/input/main.tsp @@ -0,0 +1,25 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; + @field(2) type: InputType; +} + +enum InputType { + FOO: 1, + BAR: 2, +} + +model Output { + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/enum/input/main.tsp b/packages/protobuf/test/scenarios/enum/input/main.tsp new file mode 100644 index 00000000000..e40d8df990f --- /dev/null +++ b/packages/protobuf/test/scenarios/enum/input/main.tsp @@ -0,0 +1,33 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; + @field(2) type: InputType; + @field(3) aliased: InputTypeWithAlias; +} + +enum InputType { + FOO: 0, + BAR: 1, +} + +enum InputTypeWithAlias { + BAZ: 0, + QUX: 1, + FUZ: 1, +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/enum/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/enum/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..9d8ae850a0b --- /dev/null +++ b/packages/protobuf/test/scenarios/enum/output/@typespec/protobuf/main.proto @@ -0,0 +1,31 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +enum InputType { + FOO = 0; + BAR = 1; +} + +enum InputTypeWithAlias { + option allow_alias = true; + + BAZ = 0; + QUX = 1; + FUZ = 1; +} + +message Input { + string testInputField = 1; + InputType type = 2; + InputTypeWithAlias aliased = 3; +} + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Foo(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/extern/input/main.tsp b/packages/protobuf/test/scenarios/extern/input/main.tsp new file mode 100644 index 00000000000..2d61b90d8ff --- /dev/null +++ b/packages/protobuf/test/scenarios/extern/input/main.tsp @@ -0,0 +1,17 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...MyExtern): WellKnown.Empty; + + bar(@field(1) empty: WellKnown.Empty): { + @field(1) myExtern: MyExtern; + }; +} + +model MyExtern is Extern<"foo/bar.proto", "foo.Bar">; diff --git a/packages/protobuf/test/scenarios/extern/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/extern/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..119d0339a86 --- /dev/null +++ b/packages/protobuf/test/scenarios/extern/output/@typespec/protobuf/main.proto @@ -0,0 +1,19 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +import "foo/bar.proto"; +import "google/protobuf/empty.proto"; + +message BarRequest { + google.protobuf.Empty empty = 1; +} + +message BarResponse { + foo.Bar myExtern = 1; +} + +service Service { + rpc Foo(foo.Bar) returns (google.protobuf.Empty); + rpc Bar(BarRequest) returns (BarResponse); +} diff --git a/packages/protobuf/test/scenarios/illegal field reservations/diagnostics.txt b/packages/protobuf/test/scenarios/illegal field reservations/diagnostics.txt new file mode 100644 index 00000000000..da65b9cb4e5 --- /dev/null +++ b/packages/protobuf/test/scenarios/illegal field reservations/diagnostics.txt @@ -0,0 +1,2 @@ +/test/.tsp/lib/lib.tsp:81:1 - error @typespec/protobuf/illegal-reservation: reservation value must be a string literal, uint32 literal, or a tuple of two uint32 literals denoting a range +/test/.tsp/lib/lib.tsp:51:1 - error @typespec/protobuf/illegal-reservation: reservation value must be a string literal, uint32 literal, or a tuple of two uint32 literals denoting a range \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/illegal field reservations/input/main.tsp b/packages/protobuf/test/scenarios/illegal field reservations/input/main.tsp new file mode 100644 index 00000000000..c57f720991d --- /dev/null +++ b/packages/protobuf/test/scenarios/illegal field reservations/input/main.tsp @@ -0,0 +1,16 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): {}; +} + +@reserve(2, 15, [9, 11], "foo", string, uint32) +model Input { + @field(1) testInputField: string; +} diff --git a/packages/protobuf/test/scenarios/inferred-message-names/input/main.tsp b/packages/protobuf/test/scenarios/inferred-message-names/input/main.tsp new file mode 100644 index 00000000000..c91c1e9048a --- /dev/null +++ b/packages/protobuf/test/scenarios/inferred-message-names/input/main.tsp @@ -0,0 +1,16 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.test", +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(@field(1) testInputField: string): { + @field(1) testOutputField: int32; + @field(2) secondField: string; + }; +} diff --git a/packages/protobuf/test/scenarios/inferred-message-names/output/@typespec/protobuf/com/azure/test.proto b/packages/protobuf/test/scenarios/inferred-message-names/output/@typespec/protobuf/com/azure/test.proto new file mode 100644 index 00000000000..6337433b26d --- /dev/null +++ b/packages/protobuf/test/scenarios/inferred-message-names/output/@typespec/protobuf/com/azure/test.proto @@ -0,0 +1,18 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.test; + +message FooRequest { + string testInputField = 1; +} + +message FooResponse { + int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Foo(FooRequest) returns (FooResponse); +} diff --git a/packages/protobuf/test/scenarios/intrinsics/input/main.tsp b/packages/protobuf/test/scenarios/intrinsics/input/main.tsp new file mode 100644 index 00000000000..3fcc60752e4 --- /dev/null +++ b/packages/protobuf/test/scenarios/intrinsics/input/main.tsp @@ -0,0 +1,17 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): void; +} + +model Input { + @field(1) testInputField: unknown; +} diff --git a/packages/protobuf/test/scenarios/intrinsics/output/@typespec/protobuf/com/azure/Test.proto b/packages/protobuf/test/scenarios/intrinsics/output/@typespec/protobuf/com/azure/Test.proto new file mode 100644 index 00000000000..292c1a06207 --- /dev/null +++ b/packages/protobuf/test/scenarios/intrinsics/output/@typespec/protobuf/com/azure/Test.proto @@ -0,0 +1,16 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.Test; + +import "google/protobuf/any.proto"; +import "google/protobuf/empty.proto"; + +message Input { + google.protobuf.Any testInputField = 1; +} + +service Service { + rpc Foo(Input) returns (google.protobuf.Empty); +} diff --git a/packages/protobuf/test/scenarios/map/input/main.tsp b/packages/protobuf/test/scenarios/map/input/main.tsp new file mode 100644 index 00000000000..b09d77d9ccb --- /dev/null +++ b/packages/protobuf/test/scenarios/map/input/main.tsp @@ -0,0 +1,15 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): WellKnown.Empty; +} + +model Input { + @field(1) testInputField: Map; +} diff --git a/packages/protobuf/test/scenarios/map/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/map/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..6907839def4 --- /dev/null +++ b/packages/protobuf/test/scenarios/map/output/@typespec/protobuf/main.proto @@ -0,0 +1,13 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +import "google/protobuf/empty.proto"; + +message Input { + map testInputField = 1; +} + +service Service { + rpc Foo(Input) returns (google.protobuf.Empty); +} diff --git a/packages/protobuf/test/scenarios/model-no-package/diagnostics.txt b/packages/protobuf/test/scenarios/model-no-package/diagnostics.txt new file mode 100644 index 00000000000..cf20bd1ef9e --- /dev/null +++ b/packages/protobuf/test/scenarios/model-no-package/diagnostics.txt @@ -0,0 +1,2 @@ +/test/main.tsp:14:12 - error @typespec/protobuf/model-not-in-package: model Input is not in a namespace that uses the '@Protobuf.package' decorator +/test/main.tsp:17:13 - error @typespec/protobuf/model-not-in-package: model Input is not in a namespace that uses the '@Protobuf.package' decorator \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/model-no-package/input/main.tsp b/packages/protobuf/test/scenarios/model-no-package/input/main.tsp new file mode 100644 index 00000000000..4cd41b90f77 --- /dev/null +++ b/packages/protobuf/test/scenarios/model-no-package/input/main.tsp @@ -0,0 +1,19 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +model Input { + @field(1) name: string; +} + +@package +namespace Test { + @Protobuf.service + interface Service { + // Reference a message type that isn't in this package. + example(@field(1) input: Input): WellKnown.Empty; + + // Spread input directly + example2(...Input): WellKnown.Any; + } +} diff --git a/packages/protobuf/test/scenarios/name-collision/input/main.tsp b/packages/protobuf/test/scenarios/name-collision/input/main.tsp new file mode 100644 index 00000000000..e8ae67325e7 --- /dev/null +++ b/packages/protobuf/test/scenarios/name-collision/input/main.tsp @@ -0,0 +1,24 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +model ExampleRequest { + @field(1) test: uint32; +} + +model ExampleResponse { + @field(1) test: string; +} + +@Protobuf.service +interface Service { + // invalid field index + example(@field(1) test: string): { + @field(1) test: uint32; + }; + + example2(...ExampleRequest): ExampleResponse; +} diff --git a/packages/protobuf/test/scenarios/name-collision/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/name-collision/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..bd7a640d097 --- /dev/null +++ b/packages/protobuf/test/scenarios/name-collision/output/@typespec/protobuf/main.proto @@ -0,0 +1,24 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +message ExampleRequest { + uint32 test = 1; +} + +message ExampleResponse { + string test = 1; +} + +message ExampleRequest { + string test = 1; +} + +message ExampleResponse { + uint32 test = 1; +} + +service Service { + rpc Example(ExampleRequest) returns (ExampleResponse); + rpc Example2(ExampleRequest) returns (ExampleResponse); +} diff --git a/packages/protobuf/test/scenarios/options-invalid/diagnostics.txt b/packages/protobuf/test/scenarios/options-invalid/diagnostics.txt new file mode 100644 index 00000000000..08e74b4b2b1 --- /dev/null +++ b/packages/protobuf/test/scenarios/options-invalid/diagnostics.txt @@ -0,0 +1 @@ +/test/main.tsp:5:10 - error invalid-argument: Argument '(anonymous model)' is not assignable to parameter of type 'TypeSpec.Protobuf.PackageDetails' \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/options-invalid/input/main.tsp b/packages/protobuf/test/scenarios/options-invalid/input/main.tsp new file mode 100644 index 00000000000..26f01df0b86 --- /dev/null +++ b/packages/protobuf/test/scenarios/options-invalid/input/main.tsp @@ -0,0 +1,25 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", + options: { + java_package: {}, + }, +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/options/input/main.tsp b/packages/protobuf/test/scenarios/options/input/main.tsp new file mode 100644 index 00000000000..427312d8324 --- /dev/null +++ b/packages/protobuf/test/scenarios/options/input/main.tsp @@ -0,0 +1,25 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", + options: { + java_package: "com.azure.test", + }, +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/options/output/@typespec/protobuf/com/azure/Test.proto b/packages/protobuf/test/scenarios/options/output/@typespec/protobuf/com/azure/Test.proto new file mode 100644 index 00000000000..f5457833f28 --- /dev/null +++ b/packages/protobuf/test/scenarios/options/output/@typespec/protobuf/com/azure/Test.proto @@ -0,0 +1,20 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.Test; + +option java_package = "com.azure.test"; + +message Input { + string testInputField = 1; +} + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Foo(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/reserved field collisions/diagnostics.txt b/packages/protobuf/test/scenarios/reserved field collisions/diagnostics.txt new file mode 100644 index 00000000000..d133a9c75f1 --- /dev/null +++ b/packages/protobuf/test/scenarios/reserved field collisions/diagnostics.txt @@ -0,0 +1,5 @@ +/test/main.tsp:15:13 - error @typespec/protobuf/field-name: field name 'foo' was reserved by a call to @reserve on this model +/test/main.tsp:16:10 - error @typespec/protobuf/field-index: field index 2 was reserved by a call to @reserve on this model +/test/main.tsp:17:10 - error @typespec/protobuf/field-index: field index 9 falls within a range reserved by a call to @reserve on this model +/test/main.tsp:18:10 - error @typespec/protobuf/field-index: field index 11 falls within a range reserved by a call to @reserve on this model +/test/main.tsp:18:14 - error @typespec/protobuf/field-name: field name 'bar' was reserved by a call to @reserve on this model \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/reserved field collisions/input/main.tsp b/packages/protobuf/test/scenarios/reserved field collisions/input/main.tsp new file mode 100644 index 00000000000..4046c2b39f5 --- /dev/null +++ b/packages/protobuf/test/scenarios/reserved field collisions/input/main.tsp @@ -0,0 +1,19 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): {}; +} + +@reserve(2, 15, [9, 11], "foo", "bar") +model Input { + @field(1) foo: string; + @field(2) field2: int32; + @field(9) field9: uint32; + @field(11) bar: sint32; +} diff --git a/packages/protobuf/test/scenarios/reserved fields/input/main.tsp b/packages/protobuf/test/scenarios/reserved fields/input/main.tsp new file mode 100644 index 00000000000..2bfbc9e169d --- /dev/null +++ b/packages/protobuf/test/scenarios/reserved fields/input/main.tsp @@ -0,0 +1,16 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): {}; +} + +@reserve(2, 15, [9, 11], "foo", "bar") +model Input { + @field(1) testInputField: string; +} diff --git a/packages/protobuf/test/scenarios/reserved fields/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/reserved fields/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..478af2f97cc --- /dev/null +++ b/packages/protobuf/test/scenarios/reserved fields/output/@typespec/protobuf/main.proto @@ -0,0 +1,16 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +message Input { + reserved 2, 15, 9 to 11; + reserved "foo", "bar"; + + string testInputField = 1; +} + +message FooResponse {} + +service Service { + rpc Foo(Input) returns (FooResponse); +} diff --git a/packages/protobuf/test/scenarios/simple-error/diagnostics.txt b/packages/protobuf/test/scenarios/simple-error/diagnostics.txt new file mode 100644 index 00000000000..2341433dbeb --- /dev/null +++ b/packages/protobuf/test/scenarios/simple-error/diagnostics.txt @@ -0,0 +1,6 @@ +/test/main.tsp:12:5 - error @typespec/protobuf/field-index: field index 0 is invalid (must be an integer greater than zero) +/test/main.tsp:13:5 - error @typespec/protobuf/field-index: field index 536870912 is out of bounds (must be less than 536870912) +/test/main.tsp:14:5 - error @typespec/protobuf/field-index: field index 19000 falls within the implementation-reserved range of 19000-19999 inclusive +/test/main.tsp:15:5 - error @typespec/protobuf/field-index: field index 19123 falls within the implementation-reserved range of 19000-19999 inclusive +/test/main.tsp:16:5 - error @typespec/protobuf/field-index: field index 19999 falls within the implementation-reserved range of 19000-19999 inclusive +/test/main.tsp:21:1 - error decorator-wrong-target: Cannot apply @message decorator to Test.Test since it is not assignable to object \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/simple-error/input/main.tsp b/packages/protobuf/test/scenarios/simple-error/input/main.tsp new file mode 100644 index 00000000000..f4b649246a1 --- /dev/null +++ b/packages/protobuf/test/scenarios/simple-error/input/main.tsp @@ -0,0 +1,22 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + // invalid field index + invalidIndices( + @field(0) testInputField: string, + @field(536870912) testInputField2: string, + @field(19000) testInputField3: string, + @field(19123) testInputField4: string, + @field(19999) testInputField5: string + ): {}; +} + +// Cannot apply @message to interface +@message +interface Test {} diff --git a/packages/protobuf/test/scenarios/simple-no-service/input/main.tsp b/packages/protobuf/test/scenarios/simple-no-service/input/main.tsp new file mode 100644 index 00000000000..c70c2060d13 --- /dev/null +++ b/packages/protobuf/test/scenarios/simple-no-service/input/main.tsp @@ -0,0 +1,22 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", +}) +namespace Test; + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} + +@message +model ExplicitlyDeclared { + @field(2) testField: string; +} diff --git a/packages/protobuf/test/scenarios/simple-no-service/output/@typespec/protobuf/com/azure/Test.proto b/packages/protobuf/test/scenarios/simple-no-service/output/@typespec/protobuf/com/azure/Test.proto new file mode 100644 index 00000000000..2aeda7b5782 --- /dev/null +++ b/packages/protobuf/test/scenarios/simple-no-service/output/@typespec/protobuf/com/azure/Test.proto @@ -0,0 +1,18 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.Test; + +message Input { + string testInputField = 1; +} + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} + +message ExplicitlyDeclared { + string testField = 2; +} diff --git a/packages/protobuf/test/scenarios/simple/input/main.tsp b/packages/protobuf/test/scenarios/simple/input/main.tsp new file mode 100644 index 00000000000..d8c0c005367 --- /dev/null +++ b/packages/protobuf/test/scenarios/simple/input/main.tsp @@ -0,0 +1,22 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package({ + name: "com.azure.Test", +}) +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/simple/output/@typespec/protobuf/com/azure/Test.proto b/packages/protobuf/test/scenarios/simple/output/@typespec/protobuf/com/azure/Test.proto new file mode 100644 index 00000000000..32114a3b14a --- /dev/null +++ b/packages/protobuf/test/scenarios/simple/output/@typespec/protobuf/com/azure/Test.proto @@ -0,0 +1,18 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +package com.azure.Test; + +message Input { + string testInputField = 1; +} + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Foo(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/streams/input/main.tsp b/packages/protobuf/test/scenarios/streams/input/main.tsp new file mode 100644 index 00000000000..4ce0b3191d6 --- /dev/null +++ b/packages/protobuf/test/scenarios/streams/input/main.tsp @@ -0,0 +1,30 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + @stream(StreamMode.Duplex) + duplex(...Input): Output; + + @stream(StreamMode.In) + in(...Input): Output; + + @stream(StreamMode.Out) + out(...Input): Output; + + @stream(StreamMode.None) + none(...Input): Output; +} + +model Input { + @field(1) testInputField: string; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/test/scenarios/streams/output/@typespec/protobuf/main.proto b/packages/protobuf/test/scenarios/streams/output/@typespec/protobuf/main.proto new file mode 100644 index 00000000000..31181e2591d --- /dev/null +++ b/packages/protobuf/test/scenarios/streams/output/@typespec/protobuf/main.proto @@ -0,0 +1,19 @@ +/* Generated by Microsoft TypeSpec */ + +syntax = "proto3"; + +message Input { + string testInputField = 1; +} + +message Output { + int32 testOutputField = 1; + string secondField = 2; +} + +service Service { + rpc Duplex(stream Input) returns (stream Output); + rpc In(stream Input) returns (Output); + rpc Out(Input) returns (stream Output); + rpc None(Input) returns (Output); +} diff --git a/packages/protobuf/test/scenarios/type-validation/diagnostics.txt b/packages/protobuf/test/scenarios/type-validation/diagnostics.txt new file mode 100644 index 00000000000..7fdfc7c127d --- /dev/null +++ b/packages/protobuf/test/scenarios/type-validation/diagnostics.txt @@ -0,0 +1 @@ +/test/main.tsp:10:20 - error @typespec/protobuf/unsupported-return-type: Protobuf methods must return a named Model \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/type-validation/input/main.tsp b/packages/protobuf/test/scenarios/type-validation/input/main.tsp new file mode 100644 index 00000000000..cb6a5aeb8c2 --- /dev/null +++ b/packages/protobuf/test/scenarios/type-validation/input/main.tsp @@ -0,0 +1,11 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + invalidReturn(): int32; +} diff --git a/packages/protobuf/test/scenarios/union/diagnostics.txt b/packages/protobuf/test/scenarios/union/diagnostics.txt new file mode 100644 index 00000000000..7f3697fa282 --- /dev/null +++ b/packages/protobuf/test/scenarios/union/diagnostics.txt @@ -0,0 +1 @@ +/test/main.tsp:14:3 - error @typespec/protobuf/unsupported-field-type: a message field's type may not be a union \ No newline at end of file diff --git a/packages/protobuf/test/scenarios/union/input/main.tsp b/packages/protobuf/test/scenarios/union/input/main.tsp new file mode 100644 index 00000000000..76ac3fdd8f7 --- /dev/null +++ b/packages/protobuf/test/scenarios/union/input/main.tsp @@ -0,0 +1,33 @@ +import "@typespec/protobuf"; + +using TypeSpec.Protobuf; + +@package +namespace Test; + +@Protobuf.service +interface Service { + foo(...Input): Output; +} + +model Input { + @field(1) value: U; +} + +model InputA { + @field(1) testInputField: string; +} + +union U { + a: InputA, + b: InputB, +} + +model InputB { + @field(1) testInputField: uint32; +} + +model Output { + @field(1) testOutputField: int32; + @field(2) secondField: string; +} diff --git a/packages/protobuf/tsconfig.json b/packages/protobuf/tsconfig.json new file mode 100644 index 00000000000..3723ffd27d0 --- /dev/null +++ b/packages/protobuf/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../tsconfig.json", + "references": [ + { "path": "../compiler/tsconfig.json" }, + { "path": "../rest/tsconfig.json" }, + { "path": "../openapi/tsconfig.json" } + ], + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo", + "types": ["node", "mocha"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/packages/website/.scripts/regen-ref-docs.mjs b/packages/website/.scripts/regen-ref-docs.mjs index c44beb31844..f763dfa192b 100644 --- a/packages/website/.scripts/regen-ref-docs.mjs +++ b/packages/website/.scripts/regen-ref-docs.mjs @@ -33,6 +33,13 @@ await generateLibraryDocs( join(repoRoot, "docs/standard-library/openapi/reference") ); +// Protobuf +await generateLibraryDocs( + join(repoRoot, "packages/protobuf"), + ["TypeSpec.Protobuf"], + join(repoRoot, "docs/standard-library/protobuf/reference") +); + // Versioning await generateLibraryDocs( join(repoRoot, "packages/versioning"), diff --git a/packages/website/package.json b/packages/website/package.json index 34c15914441..0b07a78e807 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -36,6 +36,7 @@ "@typespec/http": "~0.43.1", "@typespec/rest": "~0.43.0", "@typespec/openapi": "~0.43.0", + "@typespec/protobuf": "~0.43.0", "@typespec/versioning": "~0.43.0", "@docusaurus/module-type-aliases": "^2.2.0", "@docusaurus/types": "^2.2.0", diff --git a/packages/website/sidebars.js b/packages/website/sidebars.js index 131addfe558..091014a0faa 100644 --- a/packages/website/sidebars.js +++ b/packages/website/sidebars.js @@ -111,6 +111,14 @@ const sidebars = { "standard-library/openapi/openapi", ], }, + { + type: "category", + label: "Protobuf", + items: [ + "standard-library/protobuf/overview", + createLibraryReferenceStructure("protobuf"), + ], + }, { type: "category", label: "Versioning", diff --git a/rush.json b/rush.json index 32d6b3bcfb3..28104290230 100644 --- a/rush.json +++ b/rush.json @@ -187,6 +187,12 @@ "reviewCategory": "production", "shouldPublish": false }, + { + "packageName": "@typespec/protobuf", + "projectFolder": "packages/protobuf", + "reviewCategory": "production", + "shouldPublish": false + }, { "packageName": "@typespec/ref-doc", "projectFolder": "packages/ref-doc",