feat!: optimize payload size of payload v37

docs/rfc/000-Payload-v37.md

@@ -123,9 +123,7 @@ The implementation MUST NOT fail if the algorithm is not supported.
 
 This field represents the public key of the author.
 
-The value is in the DER encoding of the SubjectPublicKeyInfo (`spki`) structure from [RFC 5280][rfc5280].
-
-[rfc5280]: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.7
+The value is in the compressed format of the EC key.
 
 When it is `null`, it represents no this information is available (due to software defeat or user choice to opt-out).
 
@@ -155,7 +153,7 @@ type PublicEncrypted = [kind: EncryptionKind.Public, AES_KEY: AES_KEY, iv: Binar
 
 ###### `AES_KEY`
 
-This field represents the AES key of this payload.
+This field represents the raw AES-256-GCM key (binary) of this payload.
 
 ###### `iv` field
 
@@ -182,7 +180,7 @@ This field represents the iv used to encrypt the message.
 
 ###### `authorEphemeralPublicKey` field
 
-This field is a Map of the DER encoding of the SubjectPublicKeyInfo (`spki`) structure from [RFC5280].
+This field is a Map of the compressed format of the EC key.
 
 The key indicates its format.
 
@@ -199,68 +197,10 @@ The implementation MUST fail when the decryption result is NOT a valid TypedMess
 
 ### `secp256k1`
 
-When `spki` is mentioned in this spec, the implementation MUST be able to recognize the SubjectPublicKeyInfo of the curve [`secp256k1`][secp256k1]. This curve is widely used in the Mask Network.
-
 [secp256k1]: https://en.bitcoin.it/wiki/Secp256k1
 
-Here is an example of the `secp256k1` public key in Binary.
-
-```plaintext
-[
-   48,  86,  48,  16,   6,   7,  42, 134,  72, 206,  61,   2,
-    1,   6,   5,  43, 129,   4,   0,  10,   3,  66,   0,   4,
-  236,  81,   1, 232, 133,  60, 235, 215, 107, 253, 124,  90,
-   12,  21,  14, 139, 178, 143, 232,  52, 240, 119, 105,  91,
-  196, 232,  84,  33, 238,  69,  42, 104, 223, 226,  96, 216,
-  191, 166,  10,  63, 179, 111, 125,  99, 161, 131, 168, 172,
-  181, 245, 168, 182, 150,  19, 182, 240, 202,  62, 202, 219,
-   21, 175, 144, 205
-]
-```
-
-### `AES_KEY`
-
-```typescript
-type AES_KEY = [alg: String, k: String]
-```
-
-This type is used in this specification to represent section 6.4 of a [JsonWebKey][rfc7518] of [AES family key][rfc7518-aes-family-key].
-
-[rfc7518]: https://datatracker.ietf.org/doc/html/rfc7518
-[rfc7518-aes-family-key]: https://datatracker.ietf.org/doc/html/rfc7518#section-6.4
-
-The implementation MUST fail when the `alg` is not recognized as a known algorithm.
-The implementation MUST fail when the `k` is not valid for the given `alg`.
-
-When encrypting with AES key, the implementation MUST NOT use `additionalData`, the `tagLength` MUST be 128.
-
-#### Encoding from JsonWebKey `jwk`
-
-```js
-function fromJsonWebKey(jwk) {
-  return [jwk.alg, jwk.k]
-}
-```
-
-#### Decoding from `key`
-
-```js
-function toJsonWebKey(key) {
-  const k = { ext: true, key_ops: ['encrypt', 'decrypt'], kty: 'oct' }
-  k.alg = key[0]
-  k.k = key[1]
-  return k
-}
-```
-
 ## FAQ
 
 ### Why the version number is negative?
 
 The pre 1.0 version of the Mask Network extension uses `-42` as its initial payload version. The number `42` comes from the book _The Hitchhiker's Guide to the Galaxy_ and the minus sign indicates this is an early version. When a new payload format is drafted, it's a natural idea that the version number should add by 1, therefore it should be `-41`. At the time of this RFC written, the latest payload is version `-38`, therefore this RFC follows the convention to mark the version as `-37`.
-
-### Why not uses the `raw` format defined in the Web Crypto specification for AES key?
-
-According to [the Web Crypto specification][webcrypto], `raw` format is NOT standardized therefore it might have a co-operational problem.
-
-[webcrypto]: https://w3c.github.io/webcrypto/#dfn-CryptoKey-slot-handle

packages/encryption/src/encryption/Decryption.ts

@@ -2,8 +2,8 @@ import { unreachable } from '@dimensiondev/kit'
 import { decodeTypedMessageFromDocument, decodeTypedMessageV38ToV40Format, TypedMessage } from '@masknet/typed-message'
 import { AESCryptoKey, EC_Public_CryptoKey, andThenAsync } from '@masknet/shared-base'
 import { None, Result } from 'ts-results'
-import { AESAlgorithmEnum, PayloadParseResult } from '../payload'
-import { decryptWithAES, importAESFromJWK } from '../utils'
+import type { PayloadParseResult } from '../payload'
+import { decryptWithAES, importAES } from '../utils'
 import {
     DecryptOptions,
     DecryptIO,
@@ -40,13 +40,7 @@ export async function* decrypt(options: DecryptOptions, io: DecryptIO): AsyncIte
         if (AESKey.err) return yield new DecryptError(ErrorReasons.PayloadBroken, AESKey.val)
         if (iv.err) return yield new DecryptError(ErrorReasons.PayloadBroken, iv.val)
         // Not calling setPostCache here. It's public post and saving key is wasting storage space.
-        return yield* decryptWithPostAESKey(
-            version,
-            AESKey.val.key as AESCryptoKey,
-            iv.val,
-            encrypted,
-            options.onDecrypted,
-        )
+        return yield* decryptWithPostAESKey(version, AESKey.val, iv.val, encrypted, options.onDecrypted)
     } else if (encryption.type === 'E2E') {
         const { iv: _iv, ownersAESKeyEncrypted } = encryption
         if (_iv.err) return yield new DecryptError(ErrorReasons.PayloadBroken, _iv.val)
@@ -202,11 +196,11 @@ async function* decryptByECDH(
                 : await derive(ephemeralPublicKey).then((aesArr) => aesArr.map((aes) => [aes, iv] as const))
         for (const [derivedKey, derivedKeyNewIV] of derivedKeys) {
             const possiblePostKey = await andThenAsync(
-                decryptWithAES(AESAlgorithmEnum.A256GCM, derivedKey, derivedKeyNewIV, encryptedPostKey),
+                decryptWithAES(derivedKey, derivedKeyNewIV, encryptedPostKey),
                 postKeyDecoder,
             )
             if (possiblePostKey.err) continue
-            const decrypted = await decryptWithAES(AESAlgorithmEnum.A256GCM, possiblePostKey.val, iv, encrypted)
+            const decrypted = await decryptWithAES(possiblePostKey.val, iv, encrypted)
             if (decrypted.err) continue
 
             io.setPostKeyCache(possiblePostKey.val).catch(() => {})
@@ -225,7 +219,7 @@ async function* decryptWithPostAESKey(
     encrypted: Uint8Array,
     report: ((message: TypedMessage) => void) | undefined,
 ): AsyncIterableIterator<DecryptProgress> {
-    const { err, val } = await decryptWithAES(AESAlgorithmEnum.A256GCM, postAESKey, iv, encrypted)
+    const { err, val } = await decryptWithAES(postAESKey, iv, encrypted)
     if (err) return yield new DecryptError(ErrorReasons.DecryptFailed, val)
     return yield* parseTypedMessage(version, val, report)
 }
@@ -250,7 +244,7 @@ function importAESKeyFromJWKFromTextEncoder(aes_raw: Uint8Array) {
         const aes_text = new TextDecoder().decode(aes_raw)
         const aes_jwk = JSON.parse(aes_text) as JsonWebKey
         if (!aes_jwk.key_ops!.includes('decrypt')) aes_jwk.key_ops!.push('decrypt')
-        return (await importAESFromJWK.AES_GCM_256(aes_jwk)).unwrap()
+        return (await importAES(aes_jwk)).unwrap()
     })
 }
 

packages/encryption/src/encryption/Encryption.ts

@@ -7,7 +7,7 @@ import {
     SerializableTypedMessages,
 } from '@masknet/typed-message'
 import { None, Option, Some } from 'ts-results'
-import { AESAlgorithmEnum, EC_Key, encodePayload, PayloadWellFormed } from '../payload'
+import { EC_Key, encodePayload, PayloadWellFormed } from '../payload'
 import { encryptWithAES } from '../utils'
 
 import {
@@ -33,7 +33,7 @@ export async function encrypt(options: EncryptOptions, io: EncryptIO): Promise<E
 
     const encodedMessage = encodeMessage(options.version, options.message)
     const encryptedMessage = encodedMessage
-        .then((message) => encryptWithAES(AESAlgorithmEnum.A256GCM, postKey, postIV, message))
+        .then((message) => encryptWithAES(postKey, postIV, message))
         .then((x) => x.unwrap())
 
     let encryption: PayloadWellFormed.PublicEncryption | PayloadWellFormed.EndToEndEncryption
@@ -43,7 +43,7 @@ export async function encrypt(options: EncryptOptions, io: EncryptIO): Promise<E
         encryption = {
             iv: postIV,
             type: 'public',
-            AESKey: { algr: AESAlgorithmEnum.A256GCM, key: postKey },
+            AESKey: postKey,
         }
     } else {
         const postKeyEncoded = encodePostKey(options.version, postKey)
@@ -113,7 +113,7 @@ async function e2e_v37(
             ['encrypt'],
         )
         // Note: we're reusing iv in the post encryption.
-        const encryptedPostKey = await encryptWithAES(AESAlgorithmEnum.A256GCM, aes, postIV, await postKeyEncoded)
+        const encryptedPostKey = await encryptWithAES(aes, postIV, await postKeyEncoded)
         return encryptedPostKey.unwrap()
     })
 

packages/encryption/src/encryption/v37-ecdh.ts

@@ -1,5 +1,4 @@
 import { IdentifierMap, ProfileIdentifier } from '@masknet/shared-base'
-import { AESAlgorithmEnum } from '../payload'
 import { encryptWithAES } from '../utils'
 import {
     type EncryptTargetE2E,
@@ -39,7 +38,7 @@ export async function v37_addReceiver(
                 ['encrypt'],
             )
             // Note: we're reusing iv in the post encryption.
-            const encryptedPostKey = await encryptWithAES(AESAlgorithmEnum.A256GCM, aes, iv, await postKeyEncoded)
+            const encryptedPostKey = await encryptWithAES(aes, iv, await postKeyEncoded)
             const result: EncryptionResultE2E = {
                 encryptedPostKey: encryptedPostKey.unwrap(),
                 target: id,

packages/encryption/src/encryption/v38-ecdh.ts

@@ -1,6 +1,5 @@
 import { concatArrayBuffer, decodeArrayBuffer } from '@dimensiondev/kit'
 import { AESCryptoKey, EC_Public_CryptoKey, IdentifierMap, ProfileIdentifier } from '@masknet/shared-base'
-import { AESAlgorithmEnum } from '../payload'
 import { encryptWithAES } from '../utils'
 import {
     EncryptError,
@@ -76,7 +75,7 @@ export async function v38_addReceiver(
                 receiverPublicKey.key,
                 ivToBePublished,
             )
-            const encryptedPostKey = await encryptWithAES(AESAlgorithmEnum.A256GCM, aes, iv, await postKeyEncoded)
+            const encryptedPostKey = await encryptWithAES(aes, iv, await postKeyEncoded)
             return {
                 ivToBePublished,
                 encryptedPostKey: encryptedPostKey.unwrap(),

packages/encryption/src/index.ts

@@ -1,11 +1,9 @@
 export {
     encodePayload,
     parsePayload,
-    AESAlgorithmEnum,
     EC_KeyCurveEnum,
     SocialNetworkEnum,
     SocialNetworkEnumToProfileDomain,
-    type AESKey,
     type EC_Key,
     type Signature,
     type PayloadParseResult,
@@ -67,4 +65,4 @@ export {
 } from './image-steganography'
 
 // TODO: remove them in the future
-export { importEC_Key, importAESFromJWK } from './utils'
+export { importEC_Key, importAES as importAESFromJWK } from './utils'

packages/encryption/src/payload/types.ts

@@ -1,6 +1,13 @@
 // This file contains normalized Payload.
 // Internal payload should not be exported
-import { ProfileIdentifier, CheckedError, OptionalResult, EC_CryptoKey, EnhanceableSite } from '@masknet/shared-base'
+import {
+    ProfileIdentifier,
+    CheckedError,
+    OptionalResult,
+    EC_CryptoKey,
+    EnhanceableSite,
+    AESCryptoKey,
+} from '@masknet/shared-base'
 import type { Result, Option } from 'ts-results'
 import type { CryptoException, PayloadException } from '../types'
 
@@ -35,7 +42,7 @@ export declare namespace PayloadParseResult {
      */
     export interface PublicEncryption {
         readonly type: 'public'
-        readonly AESKey: RequiredField<AESKey, CryptoException>
+        readonly AESKey: RequiredField<AESCryptoKey, CryptoException>
         readonly iv: RequiredField<Uint8Array>
     }
     /**
@@ -71,8 +78,8 @@ export declare namespace PayloadWellFormed {
      */
     export interface PublicEncryption {
         readonly type: 'public'
-        /** The key used to encrypt the payload. */
-        readonly AESKey: AESKey
+        /** The key used to encrypt the payload. It should be AES-256-GCM */
+        readonly AESKey: AESCryptoKey
         readonly iv: Uint8Array
     }
     /**
@@ -93,18 +100,11 @@ export interface EC_Key<K extends EC_CryptoKey = EC_CryptoKey> {
     readonly algr: EC_KeyCurveEnum
     readonly key: K
 }
-export interface AESKey {
-    readonly algr: AESAlgorithmEnum
-    readonly key: CryptoKey
-}
 export enum EC_KeyCurveEnum {
     ed25519 = 0,
     secp256p1 = 1, // P-256
     secp256k1 = 2, // K-256
 }
-export enum AESAlgorithmEnum {
-    A256GCM = 'A256GCM',
-}
 export enum SocialNetworkEnum {
     Unknown = -1,
     Facebook = 0,

packages/encryption/src/payload_internal/shared.ts

@@ -2,9 +2,9 @@ import { ProfileIdentifier, CheckedError, OptionalResult, EnhanceableSite } from
 import { Ok } from 'ts-results'
 import { PayloadParseResult, SocialNetworkEnum } from '../payload'
 import { CryptoException, PayloadException } from '../types'
-import { importAESFromJWK } from '../utils'
+import { importAES } from '../utils'
 
-const import_AES_GCM_256 = CheckedError.withErr(importAESFromJWK.AES_GCM_256, CryptoException.InvalidCryptoKey)
+const import_AES_GCM_256 = CheckedError.withErr(importAES, CryptoException.InvalidCryptoKey)
 
 /**
  * @internal

packages/encryption/src/payload_internal/version-37.encoder.ts

@@ -1,6 +1,8 @@
+import { compressSecp256k1KeyRaw } from '@masknet/shared-base'
 import { Ok } from 'ts-results'
 import type { PayloadWellFormed } from '..'
-import { encodeMessagePack, exportCryptoKeyToSPKI } from '../utils'
+import { EC_KeyCurveEnum } from '../payload/types'
+import { encodeMessagePack, exportCryptoKeyToRaw } from '../utils'
 
 const enum Index {
     version = 0,
@@ -22,37 +24,37 @@ export async function encode37(payload: PayloadWellFormed.Payload) {
     if (payload.authorPublicKey.some) {
         const { algr, key } = payload.authorPublicKey.val
         payload_arr[Index.authorPublicKeyAlgorithm] = algr
-        const spki = await exportCryptoKeyToSPKI(key)
-        if (spki.ok) {
-            payload_arr[Index.authorPublicKey] = spki.val
+        const raw = await exportCryptoKeyToRaw(key)
+        if (raw.ok) {
+            if (algr === EC_KeyCurveEnum.secp256k1)
+                payload_arr[Index.authorPublicKey] = compressSecp256k1KeyRaw(raw.val)
+            else payload_arr[Index.authorPublicKey] = raw.val
         } else {
             payload_arr[Index.authorPublicKey] = null
-            warn(key, spki.err)
+            warn(key, raw.err)
         }
     }
     if (payload.encryption.type === 'E2E') {
         const { ephemeralPublicKey, iv, ownersAESKeyEncrypted } = payload.encryption
         const keyMaterials: any = {}
         const subArr: any[] = [1, ownersAESKeyEncrypted, iv, keyMaterials]
         for (const [alg, key] of ephemeralPublicKey.entries()) {
-            const k = await exportCryptoKeyToSPKI(key)
+            const k = await exportCryptoKeyToRaw(key)
             if (k.err) warn(key, k.err)
-            else keyMaterials[alg] = k.val
+            else {
+                if (alg === EC_KeyCurveEnum.secp256k1) keyMaterials[alg] = compressSecp256k1KeyRaw(k.val)
+                else keyMaterials[alg] = k.val
+            }
         }
         payload_arr[Index.encryption] = subArr
     } else {
         const { AESKey, iv } = payload.encryption
-        const subArr = [0, [AESKey.algr, (await crypto.subtle.exportKey('jwk', AESKey.key)).k], iv]
+        const subArr = [0, new Uint8Array(await crypto.subtle.exportKey('raw', AESKey)), iv]
         payload_arr[Index.encryption] = subArr
     }
     payload_arr[Index.data] = payload.encrypted
     return Ok(encodeMessagePack(payload_arr))
 }
 function warn(key: CryptoKey, err: any) {
-    console.warn(
-        '[@masknet/encryption] Failed to encode a public key object into spki format. key is',
-        key,
-        'and the error is',
-        err,
-    )
+    console.warn('[@masknet/encryption] Failed to export public key. key is', key, 'and the error is', err)
 }