diff --git a/src/Decoder.ts b/src/Decoder.ts index adf6101..f962776 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -33,10 +33,50 @@ export type DecoderOptions = Readonly< * * This is useful if the strings may contain invalid UTF-8 sequences. * - * Note that this option only applies to string values, not map keys. Additionally, when - * enabled, raw string length is limited by the maxBinLength option. + * When enabled, raw string length is limited by the maxBinLength option. + * + * Note that this option only applies to string values, not map keys. See `rawBinaryStringKeys` + * for map keys. + */ + rawBinaryStringValues: boolean; + + /** + * By default, map keys will be decoded as UTF-8 strings. However, if this option is true, map + * keys will be returned as Uint8Arrays without additional decoding. + * + * Requires `useMap` to be true, since plain objects do not support binary keys. + * + * When enabled, raw string length is limited by the maxBinLength option. + * + * Note that this option only applies to map keys, not string values. See `rawBinaryStringValues` + * for string values. + */ + rawBinaryStringKeys: boolean; + + /** + * If true, the decoder will use the Map object to store map values. If false, it will use plain + * objects. Defaults to false. + * + * Besides the type of container, the main difference is that Map objects support a wider range + * of key types. Plain objects only support string keys (though you can enable + * `supportObjectNumberKeys` to coerce number keys to strings), while Map objects support + * strings, numbers, bigints, and Uint8Arrays. + */ + useMap: boolean; + + /** + * If true, the decoder will support decoding numbers as map keys on plain objects. Defaults to + * false. + * + * Note that any numbers used as object keys will be converted to strings, so there is a risk of + * key collision as well as the inability to re-encode the object to the same representation. + * + * This option is ignored if `useMap` is true. + * + * This is useful for backwards compatibility before `useMap` was introduced. Consider instead + * using `useMap` for new code. */ - useRawBinaryStrings: boolean; + supportObjectNumberKeys: boolean; /** * Maximum string length. @@ -82,18 +122,22 @@ const STATE_ARRAY = "array"; const STATE_MAP_KEY = "map_key"; const STATE_MAP_VALUE = "map_value"; -type MapKeyType = string | number; +type MapKeyType = string | number | bigint | Uint8Array; -const isValidMapKeyType = (key: unknown): key is MapKeyType => { - return typeof key === "string" || typeof key === "number"; -}; +function isValidMapKeyType(key: unknown, useMap: boolean, supportObjectNumberKeys: boolean): key is MapKeyType { + if (useMap) { + return typeof key === "string" || typeof key === "number" || typeof key === "bigint" || key instanceof Uint8Array; + } + // Plain objects support a more limited set of key types + return typeof key === "string" || (supportObjectNumberKeys && typeof key === "number"); +} type StackMapState = { type: typeof STATE_MAP_KEY | typeof STATE_MAP_VALUE; size: number; key: MapKeyType | null; readCount: number; - map: Record; + map: Record | Map; }; type StackArrayState = { @@ -107,6 +151,8 @@ class StackPool { private readonly stack: Array = []; private stackHeadPosition = -1; + constructor(private readonly useMap: boolean) {} + public get length(): number { return this.stackHeadPosition + 1; } @@ -130,7 +176,7 @@ class StackPool { state.type = STATE_MAP_KEY; state.readCount = 0; state.size = size; - state.map = {}; + state.map = this.useMap ? new Map() : {}; } private getUninitializedStateFromPool() { @@ -213,7 +259,10 @@ export class Decoder { private readonly extensionCodec: ExtensionCodecType; private readonly context: ContextType; private readonly intMode: IntMode; - private readonly useRawBinaryStrings: boolean; + private readonly rawBinaryStringValues: boolean; + private readonly rawBinaryStringKeys: boolean; + private readonly useMap: boolean; + private readonly supportObjectNumberKeys: boolean; private readonly maxStrLength: number; private readonly maxBinLength: number; private readonly maxArrayLength: number; @@ -227,20 +276,29 @@ export class Decoder { private view = EMPTY_VIEW; private bytes = EMPTY_BYTES; private headByte = HEAD_BYTE_REQUIRED; - private readonly stack = new StackPool(); + private readonly stack: StackPool; public constructor(options?: DecoderOptions) { this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType); this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined this.intMode = options?.intMode ?? (options?.useBigInt64 ? IntMode.AS_ENCODED : IntMode.UNSAFE_NUMBER); - this.useRawBinaryStrings = options?.useRawBinaryStrings ?? false; + this.rawBinaryStringValues = options?.rawBinaryStringValues ?? false; + this.rawBinaryStringKeys = options?.rawBinaryStringKeys ?? false; + this.useMap = options?.useMap ?? false; + this.supportObjectNumberKeys = options?.supportObjectNumberKeys ?? false; this.maxStrLength = options?.maxStrLength ?? UINT32_MAX; this.maxBinLength = options?.maxBinLength ?? UINT32_MAX; this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX; this.maxMapLength = options?.maxMapLength ?? UINT32_MAX; this.maxExtLength = options?.maxExtLength ?? UINT32_MAX; this.keyDecoder = options?.keyDecoder !== undefined ? options.keyDecoder : sharedCachedKeyDecoder; + + if (this.rawBinaryStringKeys && !this.useMap) { + throw new Error("rawBinaryStringKeys is only supported when useMap is true"); + } + + this.stack = new StackPool(this.useMap); } private reinitializeState() { @@ -404,7 +462,7 @@ export class Decoder { this.complete(); continue DECODE; } else { - object = {}; + object = this.useMap ? new Map() : {}; } } else if (headByte < 0xa0) { // fixarray (1001 xxxx) 0x90 - 0x9f @@ -571,10 +629,15 @@ export class Decoder { continue DECODE; } } else if (state.type === STATE_MAP_KEY) { - if (!isValidMapKeyType(object)) { - throw new DecodeError("The type of key must be string or number but " + typeof object); + if (!isValidMapKeyType(object, this.useMap, this.supportObjectNumberKeys)) { + const acceptableTypes = this.useMap + ? "string, number, bigint, or Uint8Array" + : this.supportObjectNumberKeys + ? "string or number" + : "string"; + throw new DecodeError(`The type of key must be ${acceptableTypes} but got ${typeof object}`); } - if (object === "__proto__") { + if (!this.useMap && object === "__proto__") { throw new DecodeError("The key __proto__ is not allowed"); } @@ -584,7 +647,11 @@ export class Decoder { } else { // it must be `state.type === State.MAP_VALUE` here - state.map[state.key!] = object; + if (this.useMap) { + (state.map as Map).set(state.key!, object); + } else { + (state.map as Record)[state.key as string] = object; + } state.readCount++; if (state.readCount === state.size) { @@ -650,10 +717,10 @@ export class Decoder { } private decodeString(byteLength: number, headerOffset: number): string | Uint8Array { - if (!this.useRawBinaryStrings || this.stateIsMapKey()) { - return this.decodeUtf8String(byteLength, headerOffset); + if (this.stateIsMapKey() ? this.rawBinaryStringKeys : this.rawBinaryStringValues) { + return this.decodeBinary(byteLength, headerOffset); } - return this.decodeBinary(byteLength, headerOffset); + return this.decodeUtf8String(byteLength, headerOffset); } private decodeUtf8String(byteLength: number, headerOffset: number): string { diff --git a/src/Encoder.ts b/src/Encoder.ts index 8c774cd..5b4399e 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -1,7 +1,7 @@ import { utf8Count, utf8Encode } from "./utils/utf8"; import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; import { setInt64, setUint64 } from "./utils/int"; -import { ensureUint8Array } from "./utils/typedArrays"; +import { ensureUint8Array, compareUint8Arrays } from "./utils/typedArrays"; import type { ExtData } from "./ExtData"; import type { ContextOf } from "./context"; @@ -41,6 +41,15 @@ export type EncoderOptions = Partial< * binary is canonical and thus comparable to another encoded binary. * * Defaults to `false`. If enabled, it spends more time in encoding objects. + * + * If enabled, the encoder will throw an error if the NaN value is included in the keys of a + * map, since it is not comparable. + * + * If enabled and the keys of a map include multiple different types, each type will be sorted + * separately, and the order of the types will be as follows: + * 1. Numbers (including bigints) + * 2. Strings + * 3. Binary data */ sortKeys: boolean; @@ -321,8 +330,10 @@ export class Encoder { // this is here instead of in doEncode so that we can try encoding with an extension first, // otherwise we would break existing extensions for bigints this.encodeBigInt(object); + } else if (object instanceof Map) { + this.encodeMap(object, depth); } else if (typeof object === "object") { - this.encodeMap(object as Record, depth); + this.encodeMapObject(object as Record, depth); } else { // symbol, function and other special object come here unless extensionCodec handles them. throw new Error(`Unrecognized object: ${Object.prototype.toString.apply(object)}`); @@ -371,11 +382,11 @@ export class Encoder { } } - private countWithoutUndefined(object: Record, keys: ReadonlyArray): number { + private countWithoutUndefined(map: Map, keys: ReadonlyArray): number { let count = 0; for (const key of keys) { - if (object[key] !== undefined) { + if (map.get(key) !== undefined) { count++; } } @@ -383,13 +394,48 @@ export class Encoder { return count; } - private encodeMap(object: Record, depth: number) { - const keys = Object.keys(object); + private sortMapKeys(keys: Array): Array { + const numericKeys: Array = []; + const stringKeys: Array = []; + const binaryKeys: Array = []; + for (const key of keys) { + if (typeof key === "number") { + if (isNaN(key)) { + throw new Error("Cannot sort map keys with NaN value"); + } + numericKeys.push(key); + } else if (typeof key === "bigint") { + numericKeys.push(key); + } else if (typeof key === "string") { + stringKeys.push(key); + } else if (ArrayBuffer.isView(key)) { + binaryKeys.push(ensureUint8Array(key)); + } else { + throw new Error(`Unsupported map key type: ${Object.prototype.toString.apply(key)}`); + } + } + numericKeys.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); // Avoid using === to compare numbers and bigints + stringKeys.sort(); + binaryKeys.sort(compareUint8Arrays); + // At the moment this arbitrarily orders the keys as numeric, string, binary + return ([] as Array).concat(numericKeys, stringKeys, binaryKeys); + } + + private encodeMapObject(object: Record, depth: number) { + this.encodeMap(new Map(Object.entries(object)), depth); + } + + private encodeMap(map: Map, depth: number) { + let keys = Array.from(map.keys()); if (this.sortKeys) { - keys.sort(); + keys = this.sortMapKeys(keys); } - const size = this.ignoreUndefined ? this.countWithoutUndefined(object, keys) : keys.length; + // Map keys may encode to the same underlying value. For example, the number 3 and the bigint 3. + // This is also possible with ArrayBufferViews. We may want to introduce a new encoding option + // which checks for duplicate keys in this sense and throws an error if they are found. + + const size = this.ignoreUndefined ? this.countWithoutUndefined(map, keys) : keys.length; if (size < 16) { // fixmap @@ -407,10 +453,20 @@ export class Encoder { } for (const key of keys) { - const value = object[key]; + const value = map.get(key); if (!(this.ignoreUndefined && value === undefined)) { - this.encodeString(key); + if (typeof key === "string") { + this.encodeString(key); + } else if (typeof key === "number") { + this.encodeNumber(key); + } else if (typeof key === "bigint") { + this.encodeBigInt(key); + } else if (ArrayBuffer.isView(key)) { + this.encodeBinary(key); + } else { + throw new Error(`Unsupported map key type: ${Object.prototype.toString.apply(key)}`); + } this.doEncode(value, depth + 1); } } diff --git a/src/utils/typedArrays.ts b/src/utils/typedArrays.ts index 6e04c21..19a3e47 100644 --- a/src/utils/typedArrays.ts +++ b/src/utils/typedArrays.ts @@ -19,3 +19,14 @@ export function createDataView(buffer: ArrayLike | ArrayBufferView | Arr const bufferView = ensureUint8Array(buffer); return new DataView(bufferView.buffer, bufferView.byteOffset, bufferView.byteLength); } + +export function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number { + const length = Math.min(a.length, b.length); + for (let i = 0; i < length; i++) { + const diff = a[i]! - b[i]!; + if (diff !== 0) { + return diff; + } + } + return a.length - b.length; +} diff --git a/test/codec-bigint.test.ts b/test/codec-bigint.test.ts index 52b8f41..64d9386 100644 --- a/test/codec-bigint.test.ts +++ b/test/codec-bigint.test.ts @@ -253,24 +253,24 @@ describe("codec BigInt", () => { const encoded = encode(value, { extensionCodec }); assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); }); - }); - it("encodes and decodes 100n", () => { - const value = BigInt(100); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); - }); + it("encodes and decodes 100n", () => { + const value = BigInt(100); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); - it("encodes and decodes -100n", () => { - const value = BigInt(-100); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); - }); + it("encodes and decodes -100n", () => { + const value = BigInt(-100); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); - it("encodes and decodes MAX_SAFE_INTEGER+1", () => { - const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + it("encodes and decodes MAX_SAFE_INTEGER+1", () => { + const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); }); context("native", () => { diff --git a/test/decode-map.test.ts b/test/decode-map.test.ts new file mode 100644 index 0000000..1944227 --- /dev/null +++ b/test/decode-map.test.ts @@ -0,0 +1,161 @@ +import assert from "assert"; +import { encode, decode, DecoderOptions, IntMode } from "../src"; + +describe("decode with useMap specified", () => { + const options = { useMap: true } satisfies DecoderOptions; + + it("decodes as Map with string keys", () => { + let actual = decode(encode({}), options); + let expected: Map = new Map(); + assert.deepStrictEqual(actual, expected); + + actual = decode(encode({ a: 1 }), options); + expected = new Map([["a", 1]]); + assert.deepStrictEqual(actual, expected); + + actual = decode(encode({ a: 1, b: { c: true } }), options); + expected = new Map([ + ["a", 1], + ["b", new Map([["c", true]])], + ]); + assert.deepStrictEqual(actual, expected); + }); + + it("decodes as Map with binary keys", () => { + const input = new Map([ + [Uint8Array.from([]), 0], + [Uint8Array.from([0, 1, 2, 3]), 1], + [Uint8Array.from([4, 5, 6, 7]), 2], + ]); + const actual = decode(encode(input), options); + assert.deepStrictEqual(actual, input); + }); + + it("decodes as Map with numeric keys", () => { + const input = new Map([ + [Number.NEGATIVE_INFINITY, 0], + [Number.MIN_SAFE_INTEGER, 1], + [-100, 2], + [-0.5, 3], + [0, 4], + [1, 5], + [2, 6], + [11.11, 7], + [Number.MAX_SAFE_INTEGER, 8], + [Number.POSITIVE_INFINITY, 9], + [NaN, 10], + ]); + const actual = decode(encode(input), options); + assert.deepStrictEqual(actual, input); + }); + + context("Numeric map keys with IntMode", () => { + const input = encode( + new Map([ + [Number.NEGATIVE_INFINITY, 0], + [BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1), 1], + [Number.MIN_SAFE_INTEGER, 2], + [-100, 3], + [-0.5, 4], + [0, 5], + [1, 6], + [2, 7], + [11.11, 8], + [Number.MAX_SAFE_INTEGER, 9], + [BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), 10], + [Number.POSITIVE_INFINITY, 11], + [NaN, 12], + ]), + ); + + it("decodes with IntMode.SAFE_NUMBER", () => { + assert.throws( + () => decode(input, { ...options, intMode: IntMode.SAFE_NUMBER }), + /Mode is IntMode\.SAFE_NUMBER and value is not a safe integer/, + ); + }); + + it("decodes with IntMode.UNSAFE_NUMBER", () => { + const actual = decode(input, { ...options, intMode: IntMode.UNSAFE_NUMBER }); + // Omit integers that exceed the safe range + const expectedSubset = new Map([ + [Number.NEGATIVE_INFINITY, 0], + [Number.MIN_SAFE_INTEGER, 2], + [-100, 3], + [-0.5, 4], + [0, 5], + [1, 6], + [2, 7], + [11.11, 8], + [Number.MAX_SAFE_INTEGER, 9], + [Number.POSITIVE_INFINITY, 11], + [NaN, 12], + ]); + assert.ok(actual instanceof Map); + assert.strictEqual(actual.size, expectedSubset.size + 2); + for (const [key, value] of expectedSubset) { + assert.deepStrictEqual(actual.get(key), value); + } + }); + + it("decodes with IntMode.MIXED", () => { + const actual = decode(input, { ...options, intMode: IntMode.MIXED }); + const expected = new Map([ + [Number.NEGATIVE_INFINITY, 0], + [BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1), 1], + [Number.MIN_SAFE_INTEGER, 2], + [-100, 3], + [-0.5, 4], + [0, 5], + [1, 6], + [2, 7], + [11.11, 8], + [Number.MAX_SAFE_INTEGER, 9], + [BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), 10], + [Number.POSITIVE_INFINITY, 11], + [NaN, 12], + ]); + assert.deepStrictEqual(actual, expected); + }); + + it("decodes with IntMode.BIGINT", () => { + const actual = decode(input, { ...options, intMode: IntMode.BIGINT }); + const expected = new Map([ + [Number.NEGATIVE_INFINITY, BigInt(0)], + [BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1), BigInt(1)], + [BigInt(Number.MIN_SAFE_INTEGER), BigInt(2)], + [BigInt(-100), BigInt(3)], + [-0.5, BigInt(4)], + [BigInt(0), BigInt(5)], + [BigInt(1), BigInt(6)], + [BigInt(2), BigInt(7)], + [11.11, BigInt(8)], + [BigInt(Number.MAX_SAFE_INTEGER), BigInt(9)], + [BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), BigInt(10)], + [Number.POSITIVE_INFINITY, BigInt(11)], + [NaN, BigInt(12)], + ]); + assert.deepStrictEqual(actual, expected); + }); + + it("decodes with IntMode.AS_ENCODED", () => { + const actual = decode(input, { ...options, intMode: IntMode.AS_ENCODED }); + const expected = new Map([ + [Number.NEGATIVE_INFINITY, 0], + [BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1), 1], + [BigInt(Number.MIN_SAFE_INTEGER), 2], + [-100, 3], + [-0.5, 4], + [0, 5], + [1, 6], + [2, 7], + [11.11, 8], + [BigInt(Number.MAX_SAFE_INTEGER), 9], + [BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), 10], + [Number.POSITIVE_INFINITY, 11], + [NaN, 12], + ]); + assert.deepStrictEqual(actual, expected); + }); + }); +}); diff --git a/test/decode-raw-strings.test.ts b/test/decode-raw-strings.test.ts index dd6d7f8..9627a08 100644 --- a/test/decode-raw-strings.test.ts +++ b/test/decode-raw-strings.test.ts @@ -2,16 +2,16 @@ import assert from "assert"; import { encode, decode } from "../src"; import type { DecoderOptions } from "../src"; -describe("decode with useRawBinaryStrings specified", () => { - const options = { useRawBinaryStrings: true } satisfies DecoderOptions; +describe("decode with rawBinaryStringValues specified", () => { + const options = { rawBinaryStringValues: true } satisfies DecoderOptions; - it("decodes string as binary", () => { + it("decodes string values as binary", () => { const actual = decode(encode("foo"), options); const expected = Uint8Array.from([0x66, 0x6f, 0x6f]); assert.deepStrictEqual(actual, expected); }); - it("decodes invalid UTF-8 string as binary", () => { + it("decodes invalid UTF-8 string values as binary", () => { const invalidUtf8String = Uint8Array.from([ 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, @@ -25,7 +25,7 @@ describe("decode with useRawBinaryStrings specified", () => { assert.deepStrictEqual(actual, invalidUtf8String); }); - it("decodes object keys as strings", () => { + it("decodes map string keys as strings", () => { const actual = decode(encode({ key: "foo" }), options); const expected = { key: Uint8Array.from([0x66, 0x6f, 0x6f]) }; assert.deepStrictEqual(actual, expected); @@ -47,3 +47,86 @@ describe("decode with useRawBinaryStrings specified", () => { }, /max length exceeded/i); }); }); + +describe("decode with rawBinaryStringKeys specified", () => { + const options = { rawBinaryStringKeys: true, useMap: true } satisfies DecoderOptions; + + it("errors if useMap is not enabled", () => { + assert.throws(() => { + decode(encode({ key: "foo" }), { rawBinaryStringKeys: true }); + }, new Error("rawBinaryStringKeys is only supported when useMap is true")); + }); + + it("decodes map string keys as binary", () => { + const actual = decode(encode({ key: "foo" }), options); + const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), "foo"]]); + assert.deepStrictEqual(actual, expected); + }); + + it("decodes invalid UTF-8 string keys as binary", () => { + const invalidUtf8String = Uint8Array.from([ + 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, + 184, 221, 66, 188, 171, 36, 135, 121, + ]); + const encodedMap = Uint8Array.from([ + 129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, + 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 163, 97, 98, 99, + ]); + const actual = decode(encodedMap, options); + const expected = new Map([[invalidUtf8String, "abc"]]); + assert.deepStrictEqual(actual, expected); + }); + + it("decodes string values as strings", () => { + const actual = decode(encode("foo"), options); + const expected = "foo"; + assert.deepStrictEqual(actual, expected); + }); + + it("ignores maxStrLength", () => { + const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions; + + const actual = decode(encode({ foo: 1 }), lengthLimitedOptions); + const expected = new Map([[Uint8Array.from([0x66, 0x6f, 0x6f]), 1]]); + assert.deepStrictEqual(actual, expected); + }); + + it("respects maxBinLength", () => { + const lengthLimitedOptions = { ...options, maxBinLength: 1 } satisfies DecoderOptions; + + assert.throws(() => { + decode(encode({ foo: 1 }), lengthLimitedOptions); + }, /max length exceeded/i); + }); +}); + +describe("decode with rawBinaryStringKeys and rawBinaryStringValues", () => { + const options = { rawBinaryStringValues: true, rawBinaryStringKeys: true, useMap: true } satisfies DecoderOptions; + + it("errors if useMap is not enabled", () => { + assert.throws(() => { + decode(encode({ key: "foo" }), { rawBinaryStringKeys: true, rawBinaryStringValues: true }); + }, new Error("rawBinaryStringKeys is only supported when useMap is true")); + }); + + it("decodes map string keys and values as binary", () => { + const actual = decode(encode({ key: "foo" }), options); + const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), Uint8Array.from([0x66, 0x6f, 0x6f])]]); + assert.deepStrictEqual(actual, expected); + }); + + it("decodes invalid UTF-8 string keys and values as binary", () => { + const invalidUtf8String = Uint8Array.from([ + 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, + 184, 221, 66, 188, 171, 36, 135, 121, + ]); + const encodedMap = Uint8Array.from([ + 129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, + 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, + 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, + ]); + const actual = decode(encodedMap, options); + const expected = new Map([[invalidUtf8String, invalidUtf8String]]); + assert.deepStrictEqual(actual, expected); + }); +}); diff --git a/test/edge-cases.test.ts b/test/edge-cases.test.ts index ca4ad51..9115435 100644 --- a/test/edge-cases.test.ts +++ b/test/edge-cases.test.ts @@ -1,7 +1,7 @@ // kind of hand-written fuzzing data // any errors should not break Encoder/Decoder instance states import assert from "assert"; -import { encode, decodeAsync, decode, Encoder, Decoder, decodeMulti, decodeMultiStream } from "../src"; +import { encode, decodeAsync, decode, Encoder, Decoder, decodeMulti, decodeMultiStream, DecodeError } from "../src"; import { DataViewIndexOutOfBoundsError } from "../src/Decoder"; function testEncoder(encoder: Encoder): void { @@ -55,6 +55,22 @@ describe("edge cases", () => { }); }); + context("numeric map keys", () => { + const input = encode(new Map([[0, 1]])); + + it("throws error by default", () => { + assert.throws(() => decode(input), new DecodeError("The type of key must be string but got number")); + }); + + it("succeeds with supportObjectNumberKeys", () => { + // note: useMap is the preferred way to decode maps with non-string keys. + // supportObjectNumberKeys is only for backward compatibility + const actual = decode(input, { supportObjectNumberKeys: true }); + const expected = { "0": 1 }; + assert.deepStrictEqual(actual, expected); + }); + }); + context("try to decode a map with non-string keys (asynchronous)", () => { it("throws errors", async () => { const decoder = new Decoder(); diff --git a/test/encode.test.ts b/test/encode.test.ts index 6f88e4d..42db030 100644 --- a/test/encode.test.ts +++ b/test/encode.test.ts @@ -135,4 +135,222 @@ describe("encode", () => { assert.throws(() => encode(MIN_INT64_MINUS_ONE), /Bigint is too small for int64: -9223372036854775809$/); } }); + + context("Map", () => { + it("encodes string keys", () => { + const m = new Map([ + ["a", 1], + ["b", 2], + ]); + const encoded = encode(m); + const expected = Uint8Array.from([130, 161, 97, 1, 161, 98, 2]); + assert.deepStrictEqual(encoded, expected); + }); + + it("encodes number keys", () => { + const m = new Map([ + [-9, 1], + [1, 2], + [2, 3], + ]); + const encoded = encode(m); + const expected = Uint8Array.from([131, 247, 1, 1, 2, 2, 3]); + assert.deepStrictEqual(encoded, expected); + }); + + it("encodes bigint keys", () => { + const m = new Map([ + [BigInt(-9), 1], + [BigInt(1), 2], + [BigInt(2), 3], + ]); + const encoded = encode(m); + const expected = Uint8Array.from([131, 247, 1, 1, 2, 2, 3]); + assert.deepStrictEqual(encoded, expected); + }); + + it("encodes binary keys", () => { + const m = new Map([ + [Uint8Array.from([]), 1], + [Uint8Array.from([1, 2, 3, 4]), 2], + [Int32Array.from([-1, 0, 1234]), 3], + ]); + const encoded = encode(m); + const expected = Uint8Array.from([ + 131, 196, 0, 1, 196, 4, 1, 2, 3, 4, 2, 196, 12, 255, 255, 255, 255, 0, 0, 0, 0, 210, 4, 0, 0, 3, + ]); + assert.deepStrictEqual(encoded, expected); + }); + + it("errors on unsupported key types", () => { + assert.throws(() => { + encode(new Map([[null, 1]])); + }, new Error("Unsupported map key type: [object Null]")); + assert.throws(() => { + encode(new Map([[undefined, 1]])); + }, new Error("Unsupported map key type: [object Undefined]")); + assert.throws(() => { + encode(new Map([[true, 1]])); + }, new Error("Unsupported map key type: [object Boolean]")); + assert.throws(() => { + encode(new Map([[false, 1]])); + }, new Error("Unsupported map key type: [object Boolean]")); + assert.throws(() => { + encode(new Map([[{}, 1]])); + }, new Error("Unsupported map key type: [object Object]")); + assert.throws(() => { + encode(new Map([[[], 1]])); + }, new Error("Unsupported map key type: [object Array]")); + }); + + context("sortKeys", () => { + it("cannonicalizes encoded string keys", () => { + const m1 = new Map([ + ["a", 1], + ["b", 2], + ]); + const m1Encoded = encode(m1, { sortKeys: true }); + const m2 = new Map([ + ["b", 2], + ["a", 1], + ]); + const m2Encoded = encode(m2, { sortKeys: true }); + assert.deepStrictEqual(m1Encoded, m2Encoded); + + const expected = Uint8Array.from([130, 161, 97, 1, 161, 98, 2]); + assert.deepStrictEqual(m1Encoded, expected); + }); + + it("cannonicalizes encoded number keys", () => { + const m1 = new Map([ + [Number.NEGATIVE_INFINITY, 0], + [-10, 1], + [0, 2], + [0.5, 3], + [100, 4], + [Number.POSITIVE_INFINITY, 5], + ]); + const m1Encoded = encode(m1, { sortKeys: true }); + const m2 = new Map([ + [0.5, 3], + [100, 4], + [Number.POSITIVE_INFINITY, 5], + [0, 2], + [-10, 1], + [Number.NEGATIVE_INFINITY, 0], + ]); + const m2Encoded = encode(m2, { sortKeys: true }); + assert.deepStrictEqual(m1Encoded, m2Encoded); + const expected = Uint8Array.from([ + 134, 203, 255, 240, 0, 0, 0, 0, 0, 0, 0, 246, 1, 0, 2, 203, 63, 224, 0, 0, 0, 0, 0, 0, 3, 100, 4, 203, 127, + 240, 0, 0, 0, 0, 0, 0, 5, + ]); + assert.deepStrictEqual(m1Encoded, expected); + }); + + it("errors in the presence of NaN", () => { + const m = new Map([ + [NaN, 1], + [0, 2], + ]); + + assert.throws(() => { + encode(m, { sortKeys: true }); + }, new Error("Cannot sort map keys with NaN value")); + }); + + it("cannonicalizes encoded bigint keys", () => { + const m1 = new Map([ + [BigInt(-10), 1], + [BigInt(0), 2], + [BigInt(100), 3], + ]); + const m1Encoded = encode(m1, { sortKeys: true }); + const m2 = new Map([ + [BigInt(100), 3], + [BigInt(0), 2], + [BigInt(-10), 1], + ]); + const m2Encoded = encode(m2, { sortKeys: true }); + assert.deepStrictEqual(m1Encoded, m2Encoded); + + const expected = Uint8Array.from([131, 246, 1, 0, 2, 100, 3]); + assert.deepStrictEqual(m1Encoded, expected); + }); + + it("cannonicalizes encoded number and bigint keys", () => { + const m1 = new Map([ + [Number.NEGATIVE_INFINITY, 0], + [BigInt(-10), 1], + [-9, 2], + [BigInt(0), 3], + [0.5, 4], + [BigInt(100), 5], + [BigInt("0xffffffffffffffff"), 6], + [Number.POSITIVE_INFINITY, 7], + ]); + const m1Encoded = encode(m1, { sortKeys: true }); + const m2 = new Map([ + [0.5, 4], + [BigInt(100), 5], + [-9, 2], + [Number.NEGATIVE_INFINITY, 0], + [BigInt(0), 3], + [Number.POSITIVE_INFINITY, 7], + [BigInt("0xffffffffffffffff"), 6], + [BigInt(-10), 1], + ]); + const m2Encoded = encode(m2, { sortKeys: true }); + assert.deepStrictEqual(m1Encoded, m2Encoded); + + const expected = Uint8Array.from([ + 136, 203, 255, 240, 0, 0, 0, 0, 0, 0, 0, 246, 1, 247, 2, 0, 3, 203, 63, 224, 0, 0, 0, 0, 0, 0, 4, 100, 5, 207, + 255, 255, 255, 255, 255, 255, 255, 255, 6, 203, 127, 240, 0, 0, 0, 0, 0, 0, 7, + ]); + assert.deepStrictEqual(m1Encoded, expected); + }); + + it("cannonicalizes encoded binary keys", () => { + const m1 = new Map([ + [Uint8Array.from([1]), 1], + [Uint8Array.from([2]), 2], + ]); + const m1Encoded = encode(m1, { sortKeys: true }); + const m2 = new Map([ + [Uint8Array.from([2]), 2], + [Uint8Array.from([1]), 1], + ]); + const m2Encoded = encode(m2, { sortKeys: true }); + assert.deepStrictEqual(m1Encoded, m2Encoded); + + const expected = Uint8Array.from([130, 196, 1, 1, 1, 196, 1, 2, 2]); + assert.deepStrictEqual(m1Encoded, expected); + }); + + it("cannonicalizes encoded mixed keys", () => { + const m1 = new Map([ + [1, 1], + [2, 2], + ["a", 3], + ["b", 4], + [Uint8Array.from([1]), 5], + [Uint8Array.from([2]), 6], + ]); + const m1Encoded = encode(m1, { sortKeys: true }); + const m2 = new Map([ + ["b", 4], + [Uint8Array.from([2]), 6], + ["a", 3], + [1, 1], + [Uint8Array.from([1]), 5], + [2, 2], + ]); + const m2Encoded = encode(m2, { sortKeys: true }); + assert.deepStrictEqual(m1Encoded, m2Encoded); + + const expected = Uint8Array.from([134, 1, 1, 2, 2, 161, 97, 3, 161, 98, 4, 196, 1, 1, 5, 196, 1, 2, 6]); + assert.deepStrictEqual(m1Encoded, expected); + }); + }); + }); }); diff --git a/test/prototype-pollution.test.ts b/test/prototype-pollution.test.ts index bc15b63..21f2eaf 100644 --- a/test/prototype-pollution.test.ts +++ b/test/prototype-pollution.test.ts @@ -1,22 +1,31 @@ -import { throws } from "assert"; +import { throws, deepStrictEqual } from "assert"; import { encode, decode, DecodeError } from "@msgpack/msgpack"; describe("prototype pollution", () => { context("__proto__ exists as a map key", () => { - it("raises DecodeError in decoding", () => { - const o = { - foo: "bar", - }; - // override __proto__ as an enumerable property - Object.defineProperty(o, "__proto__", { - value: new Date(0), - enumerable: true, - }); - const encoded = encode(o); + const o = { + foo: "bar", + }; + // override __proto__ as an enumerable property + Object.defineProperty(o, "__proto__", { + value: new Date(0), + enumerable: true, + }); + const encoded = encode(o); + it("raises DecodeError in decoding", () => { throws(() => { decode(encoded); - }, DecodeError); + }, new DecodeError("The key __proto__ is not allowed")); + }); + + it("succeeds with useMap enabled", () => { + const decoded = decode(encoded, { useMap: true }); + const expected = new Map([ + ["foo", "bar"], + ["__proto__", new Date(0)], + ]); + deepStrictEqual(decoded, expected); }); }); });