Skip to content

Commit

Permalink
Add option to use RawBinaryString class during decoding as well
Browse files Browse the repository at this point in the history
  • Loading branch information
@jasonpaulos
jasonpaulos committed Jul 18, 2024
1 parent da60ddb commit 4b18081
Show file tree
Hide file tree
Showing 2 changed files with 200 additions and 111 deletions.
31 changes: 27 additions & 4 deletions src/Decoder.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { prettyByte } from "./utils/prettyByte";
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
import { IntMode, getInt64, getUint64, convertSafeIntegerToMode, UINT32_MAX } from "./utils/int";
import { utf8Decode } from "./utils/utf8";
import { createDataView, ensureUint8Array } from "./utils/typedArrays";
import { createDataView, ensureUint8Array, RawBinaryString } from "./utils/typedArrays";
import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder";
import { DecodeError } from "./DecodeError";
import type { ContextOf } from "./context";
Expand Down Expand Up @@ -53,6 +53,17 @@ export type DecoderOptions<ContextType = undefined> = Readonly<
*/
rawBinaryStringKeys: boolean;

/**
* If true, the decoder will use the RawBinaryString class to store raw binary strings created
* during decoding from the rawBinaryStringValues and rawBinaryStringKeys options. If false, it
* will use Uint8Arrays.
*
* Defaults to false.
*
* Has no effect if rawBinaryStringValues and rawBinaryStringKeys are both false.
*/
useRawBinaryStringClass: boolean;

/**
* If true, the decoder will use the Map object to store map values. If false, it will use plain
* objects. Defaults to false.
Expand Down Expand Up @@ -126,7 +137,13 @@ type MapKeyType = string | number | bigint | Uint8Array;

function isValidMapKeyType(key: unknown, useMap: boolean, supportObjectNumberKeys: boolean): key is MapKeyType {
if (useMap) {
return typeof key === "string" || typeof key === "number" || typeof key === "bigint" || key instanceof Uint8Array;
return (
typeof key === "string" ||
typeof key === "number" ||
typeof key === "bigint" ||
key instanceof Uint8Array ||
key instanceof RawBinaryString
);
}
// Plain objects support a more limited set of key types
return typeof key === "string" || (supportObjectNumberKeys && typeof key === "number");
Expand Down Expand Up @@ -261,6 +278,7 @@ export class Decoder<ContextType = undefined> {
private readonly intMode: IntMode;
private readonly rawBinaryStringValues: boolean;
private readonly rawBinaryStringKeys: boolean;
private readonly useRawBinaryStringClass: boolean;
private readonly useMap: boolean;
private readonly supportObjectNumberKeys: boolean;
private readonly maxStrLength: number;
Expand All @@ -285,6 +303,7 @@ export class Decoder<ContextType = undefined> {
this.intMode = options?.intMode ?? (options?.useBigInt64 ? IntMode.AS_ENCODED : IntMode.UNSAFE_NUMBER);
this.rawBinaryStringValues = options?.rawBinaryStringValues ?? false;
this.rawBinaryStringKeys = options?.rawBinaryStringKeys ?? false;
this.useRawBinaryStringClass = options?.useRawBinaryStringClass ?? false;
this.useMap = options?.useMap ?? false;
this.supportObjectNumberKeys = options?.supportObjectNumberKeys ?? false;
this.maxStrLength = options?.maxStrLength ?? UINT32_MAX;
Expand Down Expand Up @@ -716,9 +735,13 @@ export class Decoder<ContextType = undefined> {
this.stack.pushArrayState(size);
}

private decodeString(byteLength: number, headerOffset: number): string | Uint8Array {
private decodeString(byteLength: number, headerOffset: number): string | Uint8Array | RawBinaryString {
if (this.stateIsMapKey() ? this.rawBinaryStringKeys : this.rawBinaryStringValues) {
return this.decodeBinary(byteLength, headerOffset);
const decoded = this.decodeBinary(byteLength, headerOffset);
if (this.useRawBinaryStringClass) {
return new RawBinaryString(decoded);
}
return decoded;
}
return this.decodeUtf8String(byteLength, headerOffset);
}
Expand Down
280 changes: 173 additions & 107 deletions test/raw-strings.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,125 +37,191 @@ describe("encode with RawBinaryString", () => {
]);
assert.deepStrictEqual(actual, expected);
});
});

describe("decode with rawBinaryStringValues specified", () => {
const options = { rawBinaryStringValues: true } satisfies DecoderOptions;

it("decodes string values as binary", () => {
const actual = decode(encode("foo"), options);
const expected = Uint8Array.from([0x66, 0x6f, 0x6f]);
it("encodes a RawBinaryString map key and value as a string", () => {
const actual = encode(
new Map([
[
new RawBinaryString(Uint8Array.from([0x6b, 0x65, 0x79])),
new RawBinaryString(Uint8Array.from([0x66, 0x6f, 0x6f])),
],
]),
);
const expected = encode({ key: "foo" });
assert.deepStrictEqual(actual, expected);
});

it("decodes invalid UTF-8 string values as binary", () => {
const encoded = Uint8Array.from([
217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50,
176, 184, 221, 66, 188, 171, 36, 135, 121,
it("encodes an invalid UTF-8 RawBinaryString map key and value as a string", () => {
const actual = encode(new Map([[new RawBinaryString(invalidUtf8String), new RawBinaryString(invalidUtf8String)]]));
const expected = Uint8Array.from([
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247,
19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84,
121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121,
]);

const actual = decode(encoded, options);
assert.deepStrictEqual(actual, invalidUtf8String);
});

it("decodes map string keys as strings", () => {
const actual = decode(encode({ key: "foo" }), options);
const expected = { key: Uint8Array.from([0x66, 0x6f, 0x6f]) };
assert.deepStrictEqual(actual, expected);
});

it("ignores maxStrLength", () => {
const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions;

const actual = decode(encode("foo"), lengthLimitedOptions);
const expected = Uint8Array.from([0x66, 0x6f, 0x6f]);
assert.deepStrictEqual(actual, expected);
});
});

it("respects maxBinLength", () => {
const lengthLimitedOptions = { ...options, maxBinLength: 1 } satisfies DecoderOptions;

assert.throws(() => {
decode(encode("foo"), lengthLimitedOptions);
}, /max length exceeded/i);
});
describe("decode with rawBinaryStringValues specified", () => {
for (const useRawBinaryStringClass of [true, false, undefined]) {
const options = { rawBinaryStringValues: true, useRawBinaryStringClass } satisfies DecoderOptions;

const prepareExpectedBytes = (expected: Uint8Array): Uint8Array | RawBinaryString => {
if (useRawBinaryStringClass) {
return new RawBinaryString(expected);
}
return expected;
};

context(`useRawBinaryStringClass=${useRawBinaryStringClass}`, () => {
it("decodes string values as binary", () => {
const actual = decode(encode("foo"), options);
const expected = prepareExpectedBytes(Uint8Array.from([0x66, 0x6f, 0x6f]));
assert.deepStrictEqual(actual, expected);
});

it("decodes invalid UTF-8 string values as binary", () => {
const encoded = Uint8Array.from([
217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19,
50, 176, 184, 221, 66, 188, 171, 36, 135, 121,
]);

const actual = decode(encoded, options);
assert.deepStrictEqual(actual, prepareExpectedBytes(invalidUtf8String));
});

it("decodes map string keys as strings", () => {
const actual = decode(encode({ key: "foo" }), options);
const expected = { key: prepareExpectedBytes(Uint8Array.from([0x66, 0x6f, 0x6f])) };
assert.deepStrictEqual(actual, expected);
});

it("ignores maxStrLength", () => {
const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions;

const actual = decode(encode("foo"), lengthLimitedOptions);
const expected = prepareExpectedBytes(Uint8Array.from([0x66, 0x6f, 0x6f]));
assert.deepStrictEqual(actual, expected);
});

it("respects maxBinLength", () => {
const lengthLimitedOptions = { ...options, maxBinLength: 1 } satisfies DecoderOptions;

assert.throws(() => {
decode(encode("foo"), lengthLimitedOptions);
}, /max length exceeded/i);
});
});
}
});

describe("decode with rawBinaryStringKeys specified", () => {
const options = { rawBinaryStringKeys: true, useMap: true } satisfies DecoderOptions;

it("errors if useMap is not enabled", () => {
assert.throws(() => {
decode(encode({ key: "foo" }), { rawBinaryStringKeys: true });
}, new Error("rawBinaryStringKeys is only supported when useMap is true"));
});

it("decodes map string keys as binary", () => {
const actual = decode(encode({ key: "foo" }), options);
const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), "foo"]]);
assert.deepStrictEqual(actual, expected);
});

it("decodes invalid UTF-8 string keys as binary", () => {
const encodedMap = Uint8Array.from([
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247,
19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 163, 97, 98, 99,
]);
const actual = decode(encodedMap, options);
const expected = new Map([[invalidUtf8String, "abc"]]);
assert.deepStrictEqual(actual, expected);
});

it("decodes string values as strings", () => {
const actual = decode(encode("foo"), options);
const expected = "foo";
assert.deepStrictEqual(actual, expected);
});

it("ignores maxStrLength", () => {
const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions;

const actual = decode(encode({ foo: 1 }), lengthLimitedOptions);
const expected = new Map([[Uint8Array.from([0x66, 0x6f, 0x6f]), 1]]);
assert.deepStrictEqual(actual, expected);
});

it("respects maxBinLength", () => {
const lengthLimitedOptions = { ...options, maxBinLength: 1 } satisfies DecoderOptions;

assert.throws(() => {
decode(encode({ foo: 1 }), lengthLimitedOptions);
}, /max length exceeded/i);
});
for (const useRawBinaryStringClass of [true, false, undefined]) {
const options = { rawBinaryStringKeys: true, useMap: true, useRawBinaryStringClass } satisfies DecoderOptions;

const prepareExpectedBytes = (expected: Uint8Array): Uint8Array | RawBinaryString => {
if (useRawBinaryStringClass) {
return new RawBinaryString(expected);
}
return expected;
};

context(`useRawBinaryStringClass=${useRawBinaryStringClass}`, () => {
it("errors if useMap is not enabled", () => {
assert.throws(() => {
decode(encode({ key: "foo" }), { rawBinaryStringKeys: true, useRawBinaryStringClass });
}, new Error("rawBinaryStringKeys is only supported when useMap is true"));
});

it("decodes map string keys as binary", () => {
const actual = decode(encode({ key: "foo" }), options);
const expected = new Map([[prepareExpectedBytes(Uint8Array.from([0x6b, 0x65, 0x79])), "foo"]]);
assert.deepStrictEqual(actual, expected);
});

it("decodes invalid UTF-8 string keys as binary", () => {
const encodedMap = Uint8Array.from([
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174,
247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 163, 97, 98, 99,
]);
const actual = decode(encodedMap, options);
const expected = new Map([[prepareExpectedBytes(invalidUtf8String), "abc"]]);
assert.deepStrictEqual(actual, expected);
});

it("decodes string values as strings", () => {
const actual = decode(encode("foo"), options);
const expected = "foo";
assert.deepStrictEqual(actual, expected);
});

it("ignores maxStrLength", () => {
const lengthLimitedOptions = { ...options, maxStrLength: 1 } satisfies DecoderOptions;

const actual = decode(encode({ foo: 1 }), lengthLimitedOptions);
const expected = new Map([[prepareExpectedBytes(Uint8Array.from([0x66, 0x6f, 0x6f])), 1]]);
assert.deepStrictEqual(actual, expected);
});

it("respects maxBinLength", () => {
const lengthLimitedOptions = { ...options, maxBinLength: 1 } satisfies DecoderOptions;

assert.throws(() => {
decode(encode({ foo: 1 }), lengthLimitedOptions);
}, /max length exceeded/i);
});
});
}
});

describe("decode with rawBinaryStringKeys and rawBinaryStringValues", () => {
const options = { rawBinaryStringValues: true, rawBinaryStringKeys: true, useMap: true } satisfies DecoderOptions;

it("errors if useMap is not enabled", () => {
assert.throws(() => {
decode(encode({ key: "foo" }), { rawBinaryStringKeys: true, rawBinaryStringValues: true });
}, new Error("rawBinaryStringKeys is only supported when useMap is true"));
});

it("decodes map string keys and values as binary", () => {
const actual = decode(encode({ key: "foo" }), options);
const expected = new Map([[Uint8Array.from([0x6b, 0x65, 0x79]), Uint8Array.from([0x66, 0x6f, 0x6f])]]);
assert.deepStrictEqual(actual, expected);
});

it("decodes invalid UTF-8 string keys and values as binary", () => {
const invalidUtf8String = Uint8Array.from([
61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176,
184, 221, 66, 188, 171, 36, 135, 121,
]);
const encodedMap = Uint8Array.from([
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247,
19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84,
121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121,
]);
const actual = decode(encodedMap, options);
const expected = new Map([[invalidUtf8String, invalidUtf8String]]);
assert.deepStrictEqual(actual, expected);
});
for (const useRawBinaryStringClass of [true, false, undefined]) {
const options = {
rawBinaryStringValues: true,
rawBinaryStringKeys: true,
useMap: true,
useRawBinaryStringClass,
} satisfies DecoderOptions;

const prepareExpectedBytes = (expected: Uint8Array): Uint8Array | RawBinaryString => {
if (useRawBinaryStringClass) {
return new RawBinaryString(expected);
}
return expected;
};

context(`useRawBinaryStringClass=${useRawBinaryStringClass}`, () => {
it("errors if useMap is not enabled", () => {
assert.throws(() => {
decode(encode({ key: "foo" }), {
rawBinaryStringKeys: true,
rawBinaryStringValues: true,
useRawBinaryStringClass,
});
}, new Error("rawBinaryStringKeys is only supported when useMap is true"));
});

it("decodes map string keys and values as binary", () => {
const actual = decode(encode({ key: "foo" }), options);
const expected = new Map([
[
prepareExpectedBytes(Uint8Array.from([0x6b, 0x65, 0x79])),
prepareExpectedBytes(Uint8Array.from([0x66, 0x6f, 0x6f])),
],
]);
assert.deepStrictEqual(actual, expected);
});

it("decodes invalid UTF-8 string keys and values as binary", () => {
const encodedMap = Uint8Array.from([
129, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116, 105, 84, 121, 46, 122, 136, 233, 221, 15, 174,
247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121, 217, 32, 61, 180, 118, 220, 39, 166, 43, 68, 219, 116,
105, 84, 121, 46, 122, 136, 233, 221, 15, 174, 247, 19, 50, 176, 184, 221, 66, 188, 171, 36, 135, 121,
]);
const actual = decode(encodedMap, options);
const expected = new Map([[prepareExpectedBytes(invalidUtf8String), prepareExpectedBytes(invalidUtf8String)]]);
assert.deepStrictEqual(actual, expected);
});
});
}
});

0 comments on commit 4b18081

Please sign in to comment.