diff --git a/common/changes/@cadl-lang/openapi3/fix-binary-response_2021-12-10-22-56.json b/common/changes/@cadl-lang/openapi3/fix-binary-response_2021-12-10-22-56.json
new file mode 100644
index 00000000000..7a88c0ea44b
--- /dev/null
+++ b/common/changes/@cadl-lang/openapi3/fix-binary-response_2021-12-10-22-56.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@cadl-lang/openapi3",
+      "comment": "**Fix** issue with @body body: bytes producing `type: string, format: bytes` instead of `type: string, format: binary` for requests and responses",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@cadl-lang/openapi3"
+}
\ No newline at end of file
diff --git a/packages/openapi3/src/openapi.ts b/packages/openapi3/src/openapi.ts
index 94ba9508349..f33f02c7b60 100644
--- a/packages/openapi3/src/openapi.ts
+++ b/packages/openapi3/src/openapi.ts
@@ -328,14 +328,26 @@ function createOAPIEmitter(program: Program, options: OpenAPIEmitterOptions) {
     }
   }
 
+  function isBinaryPayload(body: Type, contentType: string) {
+    return (
+      body.kind === "Model" &&
+      body.name === "bytes" &&
+      contentType !== "application/json" &&
+      contentType !== "text/plain"
+    );
+  }
+
   function emitResponseObject(responseModel: Type, statusCode: string = "200") {
-    let contentType = "application/json";
+    let contentType: string = "application/json";
     if (
       responseModel.kind === "Model" &&
       !responseModel.baseModel &&
       responseModel.properties.size === 0
     ) {
-      const schema = mapCadlTypeToOpenAPI(responseModel);
+      const isBinary = isBinaryPayload(responseModel, contentType);
+      const schema = isBinary
+        ? { type: "string", format: "binary" }
+        : mapCadlTypeToOpenAPI(responseModel);
       if (schema) {
         currentEndpoint.responses[statusCode] = {
           description: getResponseDescription(responseModel, statusCode),
@@ -390,12 +402,15 @@ function createOAPIEmitter(program: Program, options: OpenAPIEmitterOptions) {
       }
     }
 
-    contentEntry.schema = getSchemaOrRef(bodyModel);
+    const isBinary = isBinaryPayload(bodyModel, contentType);
+    contentEntry.schema = isBinary
+      ? { type: "string", format: "binary" }
+      : getSchemaOrRef(bodyModel);
 
     const response: any = {
       description: getResponseDescription(responseModel, statusCode),
       content: {
-        [contentType]: contentEntry,
+        [contentType ?? "application/json"]: contentEntry,
       },
     };
     if (Object.keys(headers).length > 0) {
@@ -553,7 +568,6 @@ function createOAPIEmitter(program: Program, options: OpenAPIEmitterOptions) {
     }
     const bodyParam = bodyParams[0];
     const bodyType = bodyParam.type;
-    const bodySchema = getSchemaOrRef(bodyType);
 
     const requestBody: any = {
       description: getDoc(program, bodyParam),
@@ -567,6 +581,8 @@ function createOAPIEmitter(program: Program, options: OpenAPIEmitterOptions) {
       ? getContentTypes(contentTypeParam)
       : ["application/json"];
     for (let contentType of contentTypes) {
+      const isBinary = isBinaryPayload(bodyType, contentType);
+      const bodySchema = isBinary ? { type: "string", format: "binary" } : getSchemaOrRef(bodyType);
       const contentEntry: any = {
         schema: bodySchema,
       };
diff --git a/packages/openapi3/test/test-openapi-output.ts b/packages/openapi3/test/test-openapi-output.ts
index dddab39b33d..5d78750a85e 100644
--- a/packages/openapi3/test/test-openapi-output.ts
+++ b/packages/openapi3/test/test-openapi-output.ts
@@ -762,6 +762,112 @@ describe("openapi3: responses", () => {
       "string"
     );
   });
+
+  describe("binary responses", () => {
+    it("bytes responses should default to application/json with byte format", async () => {
+      const res = await openApiFor(
+        `
+      @route("/")
+      namespace root {
+        @get op read(): bytes;
+      }
+      `
+      );
+
+      const response = res.paths["/"].get.responses["200"];
+      ok(response);
+      ok(response.content);
+      strictEqual(response.content["application/json"].schema.type, "string");
+      strictEqual(response.content["application/json"].schema.format, "byte");
+    });
+
+    it("@body body: bytes responses default to application/json with bytes format", async () => {
+      const res = await openApiFor(
+        `
+      @route("/")
+      namespace root {
+        @get op read(): {@body body: bytes};
+      }
+      `
+      );
+
+      const response = res.paths["/"].get.responses["200"];
+      ok(response);
+      ok(response.content);
+      strictEqual(response.content["application/json"].schema.type, "string");
+      strictEqual(response.content["application/json"].schema.format, "byte");
+    });
+
+    it("@header contentType text/plain should keep format to byte", async () => {
+      const res = await openApiFor(
+        `
+      @route("/")
+      namespace root {
+        @get op read(): {@header contentType: "text/plain", @body body: bytes};
+      }
+      `
+      );
+
+      const response = res.paths["/"].get.responses["200"];
+      ok(response);
+      ok(response.content);
+      strictEqual(response.content["text/plain"].schema.type, "string");
+      strictEqual(response.content["text/plain"].schema.format, "byte");
+    });
+
+    it("@header contentType not json or text should set format to binary", async () => {
+      const res = await openApiFor(
+        `
+      @route("/")
+      namespace root {
+        @get op read(): {@header contentType: "image/png", @body body: bytes};
+      }
+      `
+      );
+
+      const response = res.paths["/"].get.responses["200"];
+      ok(response);
+      ok(response.content);
+      strictEqual(response.content["image/png"].schema.type, "string");
+      strictEqual(response.content["image/png"].schema.format, "binary");
+    });
+  });
+});
+
+describe("openapi3: request", () => {
+  describe("binary request", () => {
+    it("bytes request should default to application/json byte", async () => {
+      const res = await openApiFor(
+        `
+      @route("/")
+      namespace root {
+        @post op read(@body body: bytes): {};
+      }
+      `
+      );
+
+      const requestBody = res.paths["/"].post.requestBody;
+      ok(requestBody);
+      strictEqual(requestBody.content["application/json"].schema.type, "string");
+      strictEqual(requestBody.content["application/json"].schema.format, "byte");
+    });
+
+    it("bytes request should respect @header contentType and use binary format when not json or text", async () => {
+      const res = await openApiFor(
+        `
+      @route("/")
+      namespace root {
+        @post op read(@header contentType: "image/png", @body body: bytes): {};
+      }
+      `
+      );
+
+      const requestBody = res.paths["/"].post.requestBody;
+      ok(requestBody);
+      strictEqual(requestBody.content["image/png"].schema.type, "string");
+      strictEqual(requestBody.content["image/png"].schema.format, "binary");
+    });
+  });
 });
 
 describe("openapi3: extension decorator", () => {
diff --git a/packages/samples/binary/binary.cadl b/packages/samples/binary/binary.cadl
new file mode 100644
index 00000000000..32400a6ec9f
--- /dev/null
+++ b/packages/samples/binary/binary.cadl
@@ -0,0 +1,33 @@
+import "@cadl-lang/rest";
+
+using Cadl.Http;
+
+model HasBytes {
+  bytes: bytes;
+}
+
+model BytesBody<ContentType> {
+  @body body: bytes;
+  @header contentType: ContentType;
+}
+
+@route("/test")
+namespace BytesMethod {
+  @route("base64")
+  @post
+  op jsonWithBase64(@body body: HasBytes): HasBytes;
+
+  @route("textPlain")
+  @post
+  op textPlain(...BytesBody<"text/plain">): BytesBody<"text/plain">;
+
+  @route("binaryFile")
+  @post
+  op genericBinaryFile(
+    ...BytesBody<"application/octect-stream">
+  ): BytesBody<"application/octect-stream">;
+
+  @route("imagePng")
+  @post
+  op image(...BytesBody<"image/png">): BytesBody<"image/png">;
+}
diff --git a/packages/samples/binary/main.cadl b/packages/samples/binary/main.cadl
new file mode 100644
index 00000000000..46fd9a7a21f
--- /dev/null
+++ b/packages/samples/binary/main.cadl
@@ -0,0 +1 @@
+import "./binary.cadl";
diff --git a/packages/samples/test/output/binary/openapi.json b/packages/samples/test/output/binary/openapi.json
new file mode 100644
index 00000000000..4af335731ab
--- /dev/null
+++ b/packages/samples/test/output/binary/openapi.json
@@ -0,0 +1,140 @@
+{
+  "openapi": "3.0.0",
+  "info": {
+    "title": "(title)",
+    "version": "0000-00-00"
+  },
+  "tags": [],
+  "paths": {
+    "/test/base64": {
+      "post": {
+        "operationId": "BytesMethod_jsonWithBase64",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "A successful response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HasBytes"
+                }
+              }
+            }
+          }
+        },
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/HasBytes"
+              }
+            }
+          }
+        }
+      }
+    },
+    "/test/textPlain": {
+      "post": {
+        "operationId": "BytesMethod_textPlain",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "A successful response",
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string",
+                  "format": "byte"
+                }
+              }
+            }
+          }
+        },
+        "requestBody": {
+          "content": {
+            "text/plain": {
+              "schema": {
+                "type": "string",
+                "format": "byte"
+              }
+            }
+          }
+        }
+      }
+    },
+    "/test/binaryFile": {
+      "post": {
+        "operationId": "BytesMethod_genericBinaryFile",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "A successful response",
+            "content": {
+              "application/octect-stream": {
+                "schema": {
+                  "type": "string",
+                  "format": "binary"
+                }
+              }
+            }
+          }
+        },
+        "requestBody": {
+          "content": {
+            "application/octect-stream": {
+              "schema": {
+                "type": "string",
+                "format": "binary"
+              }
+            }
+          }
+        }
+      }
+    },
+    "/test/imagePng": {
+      "post": {
+        "operationId": "BytesMethod_image",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "A successful response",
+            "content": {
+              "image/png": {
+                "schema": {
+                  "type": "string",
+                  "format": "binary"
+                }
+              }
+            }
+          }
+        },
+        "requestBody": {
+          "content": {
+            "image/png": {
+              "schema": {
+                "type": "string",
+                "format": "binary"
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "components": {
+    "schemas": {
+      "HasBytes": {
+        "type": "object",
+        "properties": {
+          "bytes": {
+            "type": "string",
+            "format": "byte"
+          }
+        },
+        "required": [
+          "bytes"
+        ]
+      }
+    }
+  }
+}