refactor: move brother_node development artifact to dev/test-nodes subdirectory
Development Artifact Cleanup: ✅ BROTHER_NODE REORGANIZATION: Moved development test node to appropriate location - dev/test-nodes/brother_node/: Moved from root directory for better organization - Contains development configuration, test logs, and test chain data - No impact on production systems - purely development/testing artifact ✅ DEVELOPMENT ARTIFACTS IDENTIFIED: - Chain ID: aitbc-brother-chain (test/development chain) - Ports: 8010 (P2P) and 8011 (RPC) - different from production - Environment: .env file with test configuration - Logs: rpc.log and node.log from development testing session (March 15, 2026) ✅ ROOT DIRECTORY CLEANUP: Removed development clutter from production directory - brother_node/ moved to dev/test-nodes/brother_node/ - Root directory now contains only production-ready components - Development artifacts properly organized in dev/ subdirectory DIRECTORY STRUCTURE IMPROVEMENT: 📁 dev/test-nodes/: Development and testing node configurations 🏗️ Root Directory: Clean production structure with only essential components 🧪 Development Isolation: Test environments separated from production BENEFITS: ✅ Clean Production Directory: No development artifacts in root ✅ Better Organization: Development nodes grouped in dev/ subdirectory ✅ Clear Separation: Production vs development environments clearly distinguished ✅ Maintainability: Easier to identify and manage development components RESULT: Successfully moved brother_node development artifact to dev/test-nodes/ subdirectory, cleaning up the root directory while preserving development testing environment for future use.
This commit is contained in:
1
dev/env/node_modules/@streamparser/json/test/CommonJS.cjs
generated
vendored
Executable file
1
dev/env/node_modules/@streamparser/json/test/CommonJS.cjs
generated
vendored
Executable file
@@ -0,0 +1 @@
|
||||
require('@streamparser/json');
|
||||
44
dev/env/node_modules/@streamparser/json/test/bom.ts
generated
vendored
Executable file
44
dev/env/node_modules/@streamparser/json/test/bom.ts
generated
vendored
Executable file
@@ -0,0 +1,44 @@
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import { runJSONParserTest } from "./utils/testRunner.js";
|
||||
|
||||
describe("BOM", () => {
|
||||
test("should support UTF-8 BOM", () => {
|
||||
runJSONParserTest(
|
||||
new JSONParser(),
|
||||
new Uint8Array([0xef, 0xbb, 0xbf, 0x31]),
|
||||
({ value }) => expect(value).toBe(1),
|
||||
);
|
||||
});
|
||||
|
||||
test("should support UTF-16 BE BOM", () => {
|
||||
runJSONParserTest(
|
||||
new JSONParser(),
|
||||
new Uint16Array([0xfeff, 0x3131]),
|
||||
({ value }) => expect(value).toBe(11),
|
||||
);
|
||||
});
|
||||
|
||||
test("should support UTF-16 LE BOM", () => {
|
||||
runJSONParserTest(
|
||||
new JSONParser(),
|
||||
new Uint16Array([0xfffe, 0x3131]),
|
||||
({ value }) => expect(value).toBe(11),
|
||||
);
|
||||
});
|
||||
|
||||
test("should support UTF-32 BE BOM", () => {
|
||||
runJSONParserTest(
|
||||
new JSONParser(),
|
||||
new Uint32Array([0x0000feff, 0x31313131]),
|
||||
({ value }) => expect(value).toBe(1111),
|
||||
);
|
||||
});
|
||||
|
||||
test("should support UTF-32 LE BOM", () => {
|
||||
runJSONParserTest(
|
||||
new JSONParser(),
|
||||
new Uint32Array([0xfffe0000, 0x31313131]),
|
||||
({ value }) => expect(value).toBe(1111),
|
||||
);
|
||||
});
|
||||
});
|
||||
146
dev/env/node_modules/@streamparser/json/test/callbacks.ts
generated
vendored
Executable file
146
dev/env/node_modules/@streamparser/json/test/callbacks.ts
generated
vendored
Executable file
@@ -0,0 +1,146 @@
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import Tokenizer from "../src/tokenizer.js";
|
||||
import TokenParser from "../src/tokenparser.js";
|
||||
import TokenType from "../src/utils/types/tokenType.js";
|
||||
|
||||
describe("callback", () => {
|
||||
test("should error on missing onToken callback", () => {
|
||||
const p = new Tokenizer();
|
||||
|
||||
try {
|
||||
p.write('"test"');
|
||||
fail("Expected to fail(");
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
|
||||
test("should error if missing onError callback", () => {
|
||||
const p = new TokenParser();
|
||||
p.end();
|
||||
|
||||
try {
|
||||
p.write({ token: TokenType.TRUE, value: true });
|
||||
fail("Expected to fail(");
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
|
||||
test("should error on missing onValue callback", () => {
|
||||
const p = new JSONParser();
|
||||
|
||||
try {
|
||||
p.write('"test"');
|
||||
fail("Expected to fail(");
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
|
||||
test("should emit onValue callback when the onToken callback is also set", () => {
|
||||
const p = new JSONParser();
|
||||
|
||||
const onValueCb = jest.fn(() => {
|
||||
/* Do nothing */
|
||||
});
|
||||
p.onValue = onValueCb;
|
||||
p.onToken = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
|
||||
p.write('"test"');
|
||||
expect(onValueCb.mock.calls).toHaveLength(1);
|
||||
expect(
|
||||
(onValueCb.mock.calls[0] as unknown as [{ value: string }])[0].value,
|
||||
).toBe("test");
|
||||
});
|
||||
|
||||
test("should handle invalid input using the onError callback if set", () => {
|
||||
const p = new JSONParser();
|
||||
p.onValue = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
p.onError = (err) =>
|
||||
expect(err.message).toEqual(
|
||||
"Unexpected type. The `write` function only accepts Arrays, TypedArrays and Strings.",
|
||||
);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
p.write(745674 as any);
|
||||
});
|
||||
|
||||
test("should handle parsing errors using the onError callback if set", () => {
|
||||
const p = new JSONParser();
|
||||
p.onValue = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
p.onError = (err) =>
|
||||
expect(err.message).toEqual(
|
||||
'Unexpected "t" at position "2" in state ENDED',
|
||||
);
|
||||
|
||||
p.write('""test""');
|
||||
});
|
||||
|
||||
test("should handle errors on callbacks using the onError callback if set", () => {
|
||||
const p = new JSONParser();
|
||||
p.onValue = () => {
|
||||
throw new Error("Unexpected error in onValue callback");
|
||||
};
|
||||
p.onError = (err) =>
|
||||
expect(err.message).toEqual("Unexpected error in onValue callback");
|
||||
|
||||
p.write('"test"');
|
||||
});
|
||||
|
||||
test("should handle errors on callbacks using the onError callback if set (tokenizer)", () => {
|
||||
const p = new Tokenizer();
|
||||
p.onToken = () => {
|
||||
throw new Error("Unexpected error in onValue callback");
|
||||
};
|
||||
p.onError = (err) =>
|
||||
expect(err.message).toEqual("Unexpected error in onValue callback");
|
||||
|
||||
p.write('"test"');
|
||||
});
|
||||
|
||||
test("should handle processing end using the onEnd callback if set", (done) => {
|
||||
const p = new JSONParser();
|
||||
p.onValue = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
p.onEnd = () => done();
|
||||
|
||||
p.write('"test"');
|
||||
});
|
||||
|
||||
test("should use default onEnd callback if none set up", () => {
|
||||
const p = new Tokenizer();
|
||||
p.onToken = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
|
||||
p.write("1");
|
||||
p.end();
|
||||
|
||||
expect(p.isEnded).toBeTruthy();
|
||||
});
|
||||
|
||||
test("should not fail if ending while the underlying tokenizer is already ended", () => {
|
||||
const separator = "\n";
|
||||
|
||||
const p = new JSONParser({ separator });
|
||||
p.onValue = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
p.onEnd = () => {
|
||||
/* Do nothing */
|
||||
};
|
||||
|
||||
p.write("{}");
|
||||
p.end();
|
||||
|
||||
expect(p.isEnded).toBeTruthy();
|
||||
});
|
||||
});
|
||||
646
dev/env/node_modules/@streamparser/json/test/emitPartial.ts
generated
vendored
Executable file
646
dev/env/node_modules/@streamparser/json/test/emitPartial.ts
generated
vendored
Executable file
@@ -0,0 +1,646 @@
|
||||
import TokenType from "../src/utils/types/tokenType.js";
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import Tokenizer from "../src/tokenizer.js";
|
||||
import {
|
||||
TestData,
|
||||
runJSONParserTest,
|
||||
runTokenizerTest,
|
||||
} from "./utils/testRunner.js";
|
||||
|
||||
describe("Emit Partial", () => {
|
||||
describe("Tokenizer emit partial tokens", () => {
|
||||
const emitPartialTokenTestData: TestData[] = [
|
||||
{
|
||||
value: ["tr", "ue"],
|
||||
expected: [
|
||||
{ token: TokenType.TRUE, value: true, partial: true },
|
||||
{ token: TokenType.TRUE, value: true, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["t", "ru", "e"],
|
||||
expected: [
|
||||
{ token: TokenType.TRUE, value: true, partial: true },
|
||||
{ token: TokenType.TRUE, value: true, partial: true },
|
||||
{ token: TokenType.TRUE, value: true, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["f", "al", "se"],
|
||||
expected: [
|
||||
{ token: TokenType.FALSE, value: false, partial: true },
|
||||
{ token: TokenType.FALSE, value: false, partial: true },
|
||||
{ token: TokenType.FALSE, value: false, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["fal", "se"],
|
||||
expected: [
|
||||
{ token: TokenType.FALSE, value: false, partial: true },
|
||||
{ token: TokenType.FALSE, value: false, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["0", ".", "123"],
|
||||
expected: [
|
||||
{ token: TokenType.NUMBER, value: 0, partial: true },
|
||||
{ token: TokenType.NUMBER, value: 0.123, partial: true },
|
||||
{ token: TokenType.NUMBER, value: 0.123, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["n", "u", "l", "l"],
|
||||
expected: [
|
||||
{ token: TokenType.NULL, value: null, partial: true },
|
||||
{ token: TokenType.NULL, value: null, partial: true },
|
||||
{ token: TokenType.NULL, value: null, partial: true },
|
||||
{ token: TokenType.NULL, value: null, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["n", "u", "l", "l"],
|
||||
expected: [
|
||||
{ token: TokenType.NULL, value: null, partial: true },
|
||||
{ token: TokenType.NULL, value: null, partial: true },
|
||||
{ token: TokenType.NULL, value: null, partial: true },
|
||||
{ token: TokenType.NULL, value: null, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: "{",
|
||||
expected: [{ token: TokenType.LEFT_BRACE, value: "{", partial: false }],
|
||||
},
|
||||
{
|
||||
value: ['{ "fo', "o", '"', ': "', '"'],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "fo", partial: true },
|
||||
{ token: TokenType.STRING, value: "foo", partial: true },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "", partial: true },
|
||||
{ token: TokenType.STRING, value: "", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "ba', "r", '"'],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "ba", partial: true },
|
||||
{ token: TokenType.STRING, value: "bar", partial: true },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "bar"', "}"],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.RIGHT_BRACE, value: "}", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "foo": "bar" }',
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.RIGHT_BRACE, value: "}", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: [
|
||||
'{ "foo": "bar", "ba',
|
||||
"z",
|
||||
'": [',
|
||||
'{ "foo": "bar", "baz": [',
|
||||
'{ "foo": "bar", "baz": [1',
|
||||
"2",
|
||||
"3, ",
|
||||
],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "ba", partial: true },
|
||||
{ token: TokenType.STRING, value: "baz", partial: true },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.NUMBER, value: 1, partial: true },
|
||||
{ token: TokenType.NUMBER, value: 12, partial: true },
|
||||
{ token: TokenType.NUMBER, value: 123, partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "foo": "bar", "baz": [1]',
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.NUMBER, value: 1, partial: false },
|
||||
{ token: TokenType.RIGHT_BRACKET, value: "]", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "bar", ', ' "baz": [1,'],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.NUMBER, value: 1, partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "bar", "baz": [1,2', "3, 4", "5", "6] }"],
|
||||
expected: [
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.LEFT_BRACE,
|
||||
value: "{",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.STRING,
|
||||
value: "foo",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.COLON,
|
||||
value: ":",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.STRING,
|
||||
value: "bar",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.COMMA,
|
||||
value: ",",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.STRING,
|
||||
value: "baz",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.COLON,
|
||||
value: ":",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.LEFT_BRACKET,
|
||||
value: "[",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.NUMBER,
|
||||
value: 1,
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.COMMA,
|
||||
value: ",",
|
||||
partial: false,
|
||||
},
|
||||
{ token: TokenType.NUMBER, value: 2, partial: true },
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.NUMBER,
|
||||
value: 23,
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.COMMA,
|
||||
value: ",",
|
||||
partial: false,
|
||||
},
|
||||
{ token: TokenType.NUMBER, value: 4, partial: true },
|
||||
{ token: TokenType.NUMBER, value: 45, partial: true },
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.NUMBER,
|
||||
value: 456,
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.RIGHT_BRACKET,
|
||||
value: "]",
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
type: "complete",
|
||||
token: TokenType.RIGHT_BRACE,
|
||||
value: "}",
|
||||
partial: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "bar", "baz"', ": [{"],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "bar", "baz": [{ "a', '"'],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "a", partial: true },
|
||||
{ token: TokenType.STRING, value: "a", partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ['{ "foo": "bar", "baz": [{ "a": "b', '"'],
|
||||
expected: [
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "foo", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "bar", partial: false },
|
||||
{ token: TokenType.COMMA, value: ",", partial: false },
|
||||
{ token: TokenType.STRING, value: "baz", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.LEFT_BRACKET, value: "[", partial: false },
|
||||
{ token: TokenType.LEFT_BRACE, value: "{", partial: false },
|
||||
{ token: TokenType.STRING, value: "a", partial: false },
|
||||
{ token: TokenType.COLON, value: ":", partial: false },
|
||||
{ token: TokenType.STRING, value: "b", partial: true },
|
||||
{ token: TokenType.STRING, value: "b", partial: false },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
emitPartialTokenTestData.forEach(({ value, expected }) => {
|
||||
test(`Tokenizer emit partial tokens: ${value}`, () => {
|
||||
let i = 0;
|
||||
runTokenizerTest(
|
||||
new Tokenizer({ emitPartialTokens: true }),
|
||||
value,
|
||||
({ token, value, partial }) => {
|
||||
const expectedData = expected[i];
|
||||
expect(token).toEqual(expectedData.token);
|
||||
expect(value).toEqual(expectedData.value);
|
||||
expect(partial ?? false).toEqual(expectedData.partial);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
expect(i).toEqual(expected.length);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("TokenParser emit partial values", () => {
|
||||
const emitPartialValuesTestData: TestData[] = [
|
||||
{
|
||||
value: ['"a', "bc", '"'],
|
||||
expected: [
|
||||
{ value: "a", key: undefined, parent: undefined, partial: true },
|
||||
{ value: "abc", key: undefined, parent: undefined, partial: true },
|
||||
{ value: "abc", key: undefined, parent: undefined, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["12", ".34"],
|
||||
expected: [
|
||||
{ value: 12, key: undefined, parent: undefined, partial: true },
|
||||
{ value: 12.34, key: undefined, parent: undefined, partial: true },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["[", "]"],
|
||||
expected: [
|
||||
{ value: undefined, key: 0, parent: [], partial: true },
|
||||
{ value: [], key: undefined, parent: undefined, partial: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
value: ["[", '"a', "bc", '"', ",", '"def"', "]"],
|
||||
expected: [
|
||||
{ value: undefined, key: 0, parent: [], partial: true },
|
||||
{ value: "a", key: 0, parent: [], partial: true },
|
||||
{ value: "abc", key: 0, parent: [], partial: true },
|
||||
{ value: "abc", key: 0, parent: ["abc"], partial: false },
|
||||
{ value: "def", key: 1, parent: ["abc", "def"], partial: false },
|
||||
{
|
||||
value: ["abc", "def"],
|
||||
key: undefined,
|
||||
parent: undefined,
|
||||
partial: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
value: [
|
||||
"{",
|
||||
'"a',
|
||||
"bc",
|
||||
'"',
|
||||
":",
|
||||
'"def"',
|
||||
",",
|
||||
'"ghi":',
|
||||
'"jkl"',
|
||||
"}",
|
||||
],
|
||||
expected: [
|
||||
{ value: undefined, key: undefined, parent: {}, partial: true },
|
||||
{ value: undefined, key: "a", parent: {}, partial: true },
|
||||
{ value: undefined, key: "abc", parent: {}, partial: true },
|
||||
{ value: undefined, key: "abc", parent: {}, partial: true },
|
||||
{ value: "def", key: "abc", parent: { abc: "def" }, partial: false },
|
||||
{
|
||||
value: undefined,
|
||||
key: "ghi",
|
||||
parent: { abc: "def" },
|
||||
partial: true,
|
||||
},
|
||||
{
|
||||
value: "jkl",
|
||||
key: "ghi",
|
||||
parent: { abc: "def", ghi: "jkl" },
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: { abc: "def", ghi: "jkl" },
|
||||
key: undefined,
|
||||
parent: undefined,
|
||||
partial: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
value: [
|
||||
'{ "foo"',
|
||||
": ",
|
||||
'{ "foo1": "ba',
|
||||
"r",
|
||||
'" , "baz',
|
||||
'": [',
|
||||
'{ "foo2": "bar2", "baz2": [',
|
||||
'{ "foo3": "bar3", "baz3": [1',
|
||||
"2",
|
||||
"3, ",
|
||||
"3, 4",
|
||||
"5",
|
||||
"6] }",
|
||||
"] }] }}",
|
||||
],
|
||||
expected: [
|
||||
{ value: undefined, key: undefined, parent: {}, partial: true },
|
||||
{ value: undefined, key: "foo", parent: {}, partial: true },
|
||||
{ value: undefined, key: undefined, parent: {}, partial: true },
|
||||
{ value: undefined, key: "foo1", parent: {}, partial: true },
|
||||
{ value: "ba", key: "foo1", parent: {}, partial: true },
|
||||
{ value: "bar", key: "foo1", parent: {}, partial: true },
|
||||
{
|
||||
value: "bar",
|
||||
key: "foo1",
|
||||
parent: { foo1: "bar" },
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: undefined,
|
||||
key: "baz",
|
||||
parent: { foo1: "bar" },
|
||||
partial: true,
|
||||
},
|
||||
{
|
||||
value: undefined,
|
||||
key: "baz",
|
||||
parent: { foo1: "bar" },
|
||||
partial: true,
|
||||
},
|
||||
{ value: undefined, key: 0, parent: [], partial: true },
|
||||
{ value: undefined, key: undefined, parent: {}, partial: true },
|
||||
{ value: undefined, key: "foo2", parent: {}, partial: true },
|
||||
{
|
||||
value: "bar2",
|
||||
key: "foo2",
|
||||
parent: { foo2: "bar2" },
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: undefined,
|
||||
key: "baz2",
|
||||
parent: { foo2: "bar2" },
|
||||
partial: true,
|
||||
},
|
||||
{ value: undefined, key: 0, parent: [], partial: true },
|
||||
{ value: undefined, key: undefined, parent: {}, partial: true },
|
||||
{ value: undefined, key: "foo3", parent: {}, partial: true },
|
||||
{
|
||||
value: "bar3",
|
||||
key: "foo3",
|
||||
parent: { foo3: "bar3" },
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: undefined,
|
||||
key: "baz3",
|
||||
parent: { foo3: "bar3" },
|
||||
partial: true,
|
||||
},
|
||||
{ value: undefined, key: 0, parent: [], partial: true },
|
||||
{ value: 1, key: 0, parent: [], partial: true },
|
||||
{ value: 12, key: 0, parent: [], partial: true },
|
||||
{ value: 123, key: 0, parent: [123], partial: false },
|
||||
{ value: 3, key: 1, parent: [123, 3], partial: false },
|
||||
{ value: 4, key: 2, parent: [123, 3], partial: true },
|
||||
{ value: 45, key: 2, parent: [123, 3], partial: true },
|
||||
{ value: 456, key: 2, parent: [123, 3, 456], partial: false },
|
||||
{
|
||||
value: [123, 3, 456],
|
||||
key: "baz3",
|
||||
parent: { foo3: "bar3", baz3: [123, 3, 456] },
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: { foo3: "bar3", baz3: [123, 3, 456] },
|
||||
key: 0,
|
||||
parent: [{ foo3: "bar3", baz3: [123, 3, 456] }],
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: [{ foo3: "bar3", baz3: [123, 3, 456] }],
|
||||
key: "baz2",
|
||||
parent: {
|
||||
foo2: "bar2",
|
||||
baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }],
|
||||
},
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: {
|
||||
foo2: "bar2",
|
||||
baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }],
|
||||
},
|
||||
key: 0,
|
||||
parent: [
|
||||
{ foo2: "bar2", baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }] },
|
||||
],
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: [
|
||||
{ foo2: "bar2", baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }] },
|
||||
],
|
||||
key: "baz",
|
||||
parent: {
|
||||
foo1: "bar",
|
||||
baz: [
|
||||
{ foo2: "bar2", baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }] },
|
||||
],
|
||||
},
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: {
|
||||
foo1: "bar",
|
||||
baz: [
|
||||
{ foo2: "bar2", baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }] },
|
||||
],
|
||||
},
|
||||
key: "foo",
|
||||
parent: {
|
||||
foo: {
|
||||
foo1: "bar",
|
||||
baz: [
|
||||
{
|
||||
foo2: "bar2",
|
||||
baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
partial: false,
|
||||
},
|
||||
{
|
||||
value: {
|
||||
foo: {
|
||||
foo1: "bar",
|
||||
baz: [
|
||||
{
|
||||
foo2: "bar2",
|
||||
baz2: [{ foo3: "bar3", baz3: [123, 3, 456] }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
key: undefined,
|
||||
parent: undefined,
|
||||
partial: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
emitPartialValuesTestData.forEach(({ value, expected }) => {
|
||||
test(`TokenParser emit partial values: ${value}`, () => {
|
||||
let i = 0;
|
||||
runJSONParserTest(
|
||||
new JSONParser({ emitPartialTokens: true, emitPartialValues: true }),
|
||||
value,
|
||||
({ value, key, parent, partial }) => {
|
||||
const expectedData = expected[i];
|
||||
expect(value).toEqual(expectedData.value);
|
||||
expect(key).toEqual(expectedData.key);
|
||||
expect(parent).toEqual(expectedData.parent);
|
||||
expect(partial ?? false).toEqual(expectedData.partial);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
expect(i).toEqual(expected.length);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("TokenParser emit partial values only if matching paths when paths is present", () => {
|
||||
const value = ['{ "a"', ": 1,", '"b":', '{ "c":', "1 } }"];
|
||||
const expected = [
|
||||
{ value: undefined, key: "c", parent: {}, partial: true },
|
||||
{ value: 1, key: "c", parent: { c: 1 }, partial: false },
|
||||
];
|
||||
let i = 0;
|
||||
runJSONParserTest(
|
||||
new JSONParser({
|
||||
paths: ["$.b.c"],
|
||||
emitPartialTokens: true,
|
||||
emitPartialValues: true,
|
||||
}),
|
||||
value,
|
||||
({ value, key, parent, partial }) => {
|
||||
const expectedData = expected[i];
|
||||
expect(value).toEqual(expectedData.value);
|
||||
expect(key).toEqual(expectedData.key);
|
||||
expect(parent).toEqual(expectedData.parent);
|
||||
expect(partial ?? false).toEqual(expectedData.partial);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
expect(i).toEqual(expected.length);
|
||||
});
|
||||
});
|
||||
94
dev/env/node_modules/@streamparser/json/test/end.ts
generated
vendored
Executable file
94
dev/env/node_modules/@streamparser/json/test/end.ts
generated
vendored
Executable file
@@ -0,0 +1,94 @@
|
||||
import { runJSONParserTest } from "./utils/testRunner.js";
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
|
||||
describe("end", () => {
|
||||
test("should fail if writing after ending", async () => {
|
||||
const p = new JSONParser({ separator: "" });
|
||||
|
||||
await runJSONParserTest(p, ['"test"']);
|
||||
|
||||
try {
|
||||
await runJSONParserTest(p, ['"test"']);
|
||||
fail("Expected to fail!");
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
|
||||
const autoEndValues = ["2 2", "2.33456{}", "{}{}{}"];
|
||||
|
||||
autoEndValues.forEach((value) => {
|
||||
test(`should auto-end after emiting one object: ${value}`, async () => {
|
||||
const p = new JSONParser();
|
||||
|
||||
try {
|
||||
await runJSONParserTest(p, [value]);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
expect(p.isEnded).toBeTruthy();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const numberValues = [
|
||||
"0",
|
||||
"2",
|
||||
"2.33456",
|
||||
"2.33456e+1",
|
||||
"-2",
|
||||
"-2.33456",
|
||||
"-2.33456e+1",
|
||||
];
|
||||
|
||||
numberValues.forEach((numberValue) => {
|
||||
test(`should emit numbers if ending on a valid number: ${numberValue}`, async () => {
|
||||
const p = new JSONParser({ separator: "" });
|
||||
|
||||
await runJSONParserTest(p, [numberValue], ({ value }) =>
|
||||
expect(value).toEqual(JSON.parse(numberValue)),
|
||||
);
|
||||
p.end();
|
||||
|
||||
expect(p.isEnded).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
const endingFailingValues = [
|
||||
"2.",
|
||||
"2.33456e",
|
||||
"2.33456e+",
|
||||
'"asdfasd',
|
||||
"tru",
|
||||
'"fa',
|
||||
'"nul',
|
||||
"{",
|
||||
"[",
|
||||
'{ "a":',
|
||||
'{ "a": { "b": 1, ',
|
||||
'{ "a": { "b": 1, "c": 2, "d": 3, "e": 4 }',
|
||||
];
|
||||
|
||||
endingFailingValues.forEach((value) => {
|
||||
test(`should fail if ending in the middle of parsing: ${value}`, async () => {
|
||||
const p = new JSONParser();
|
||||
|
||||
try {
|
||||
await runJSONParserTest(p, [value]);
|
||||
p.end();
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("should not fail if ending waiting for a separator", async () => {
|
||||
const separator = "\n";
|
||||
const p = new JSONParser({ separator });
|
||||
|
||||
await runJSONParserTest(p, ["1", separator, "2"]);
|
||||
p.end();
|
||||
|
||||
expect(p.isEnded).toBeTruthy();
|
||||
});
|
||||
});
|
||||
53
dev/env/node_modules/@streamparser/json/test/inputs.ts
generated
vendored
Executable file
53
dev/env/node_modules/@streamparser/json/test/inputs.ts
generated
vendored
Executable file
@@ -0,0 +1,53 @@
|
||||
import { runJSONParserTest, type TestData } from "./utils/testRunner.js";
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import { charset } from "../src/utils/utf-8.js";
|
||||
|
||||
const quote = String.fromCharCode(charset.QUOTATION_MARK);
|
||||
|
||||
describe("inputs", () => {
|
||||
const testData: TestData[] = [
|
||||
{
|
||||
value: "test",
|
||||
expected: ["test"],
|
||||
},
|
||||
{
|
||||
value: new Uint8Array([116, 101, 115, 116]),
|
||||
expected: ["test"],
|
||||
},
|
||||
{
|
||||
value: new Uint16Array([25972, 29811]),
|
||||
expected: ["test"],
|
||||
},
|
||||
{
|
||||
value: new Uint32Array([1953719668]),
|
||||
expected: ["test"],
|
||||
},
|
||||
{
|
||||
value: [116, 101, 115, 116],
|
||||
expected: ["test"],
|
||||
},
|
||||
];
|
||||
|
||||
testData.forEach(({ value, expected: [expected] }) => {
|
||||
test(`write accept ${value}`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
[quote, value, quote],
|
||||
({ value }) => expect(value).toEqual(expected),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("write throw on invalid type", async () => {
|
||||
try {
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
[745674 as any],
|
||||
);
|
||||
fail("Expected to fail!");
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
41
dev/env/node_modules/@streamparser/json/test/keepStack.ts
generated
vendored
Executable file
41
dev/env/node_modules/@streamparser/json/test/keepStack.ts
generated
vendored
Executable file
@@ -0,0 +1,41 @@
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import { runJSONParserTest } from "./utils/testRunner.js";
|
||||
|
||||
describe("keepStack", () => {
|
||||
const testData = [
|
||||
{
|
||||
value: '{ "a": { "b": 1, "c": 2, "d": 3, "e": 4 } }',
|
||||
paths: ["$"],
|
||||
},
|
||||
{
|
||||
value: '{ "a": { "b": 1, "c": 2, "d": 3, "e": 4 } }',
|
||||
paths: ["$.a.*"],
|
||||
},
|
||||
{
|
||||
value: '{ "a": { "b": 1, "c": 2, "d": 3, "e": 4 } }',
|
||||
paths: ["$.a.e"],
|
||||
},
|
||||
{
|
||||
value: '{ "a": { "b": [1,2,3,4,5,6] } }',
|
||||
paths: ["$.a.b.*"],
|
||||
expected: 6,
|
||||
},
|
||||
{
|
||||
value: '[{ "a": 1 }, { "a": 2 }, { "a": 3 }]',
|
||||
paths: ["$.*"],
|
||||
},
|
||||
];
|
||||
|
||||
testData.forEach(({ value, paths }) => {
|
||||
test(`should keep parent empty if keepStack === false (${value} - ${paths})`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ paths, keepStack: false }),
|
||||
[value],
|
||||
({ parent }) => {
|
||||
if (parent === undefined) return;
|
||||
expect(Object.keys(parent).length).toEqual(0);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
69
dev/env/node_modules/@streamparser/json/test/offset.ts
generated
vendored
Executable file
69
dev/env/node_modules/@streamparser/json/test/offset.ts
generated
vendored
Executable file
@@ -0,0 +1,69 @@
|
||||
import { runTokenizerTest } from "./utils/testRunner.js";
|
||||
import { Tokenizer } from "../src/index.js";
|
||||
import TokenType from "../src/utils/types/tokenType.js";
|
||||
|
||||
const input1 = '{\n "string": "value",\n "number": 3,\n "object"';
|
||||
const input2 = ': {\n "key": "vд"\n },\n "array": [\n -1,\n 12\n ]\n ';
|
||||
const input3 = '"null": null, "true": true, "false": false, "frac": 3.14,';
|
||||
const input4 = '"escape": "\\"\\u00e1" }';
|
||||
|
||||
const offsets = [
|
||||
[0, TokenType.LEFT_BRACE],
|
||||
[4, TokenType.STRING],
|
||||
[12, TokenType.COLON],
|
||||
[14, TokenType.STRING],
|
||||
[21, TokenType.COMMA],
|
||||
[25, TokenType.STRING],
|
||||
[33, TokenType.COLON],
|
||||
[35, TokenType.NUMBER],
|
||||
[36, TokenType.COMMA],
|
||||
[40, TokenType.STRING],
|
||||
[48, TokenType.COLON],
|
||||
[50, TokenType.LEFT_BRACE],
|
||||
[54, TokenType.STRING],
|
||||
[59, TokenType.COLON],
|
||||
[61, TokenType.STRING],
|
||||
[69, TokenType.RIGHT_BRACE],
|
||||
[70, TokenType.COMMA],
|
||||
[74, TokenType.STRING],
|
||||
[81, TokenType.COLON],
|
||||
[83, TokenType.LEFT_BRACKET],
|
||||
[87, TokenType.NUMBER],
|
||||
[89, TokenType.COMMA],
|
||||
[93, TokenType.NUMBER],
|
||||
[98, TokenType.RIGHT_BRACKET],
|
||||
[102, TokenType.STRING],
|
||||
[108, TokenType.COLON],
|
||||
[110, TokenType.NULL],
|
||||
[114, TokenType.COMMA],
|
||||
[116, TokenType.STRING],
|
||||
[122, TokenType.COLON],
|
||||
[124, TokenType.TRUE],
|
||||
[128, TokenType.COMMA],
|
||||
[130, TokenType.STRING],
|
||||
[137, TokenType.COLON],
|
||||
[139, TokenType.FALSE],
|
||||
[144, TokenType.COMMA],
|
||||
[146, TokenType.STRING],
|
||||
[152, TokenType.COLON],
|
||||
[154, TokenType.NUMBER],
|
||||
[158, TokenType.COMMA],
|
||||
[159, TokenType.STRING],
|
||||
[167, TokenType.COLON],
|
||||
[169, TokenType.STRING],
|
||||
[180, TokenType.RIGHT_BRACE],
|
||||
];
|
||||
|
||||
test("offset", async () => {
|
||||
let i = 0;
|
||||
|
||||
await runTokenizerTest(
|
||||
new Tokenizer(),
|
||||
[input1, input2, input3, input4],
|
||||
({ token, offset }) => {
|
||||
expect(offset).toEqual(offsets[i][0]);
|
||||
expect(token).toEqual(offsets[i][1]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
121
dev/env/node_modules/@streamparser/json/test/performance.ts
generated
vendored
Executable file
121
dev/env/node_modules/@streamparser/json/test/performance.ts
generated
vendored
Executable file
@@ -0,0 +1,121 @@
|
||||
import { runJSONParserTest } from "./utils/testRunner.js";
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import { charset } from "../src/utils/utf-8.js";
|
||||
|
||||
const quote = String.fromCharCode(charset.QUOTATION_MARK);
|
||||
|
||||
const oneKB = 1024;
|
||||
const oneMB = 1024 * oneKB;
|
||||
const twoHundredMB = 200 * oneMB;
|
||||
const kbsIn200MBs = twoHundredMB / oneKB;
|
||||
|
||||
describe("performance", () => {
|
||||
describe("buffered parsing", () => {
|
||||
test("can handle large strings without running out of memory", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize: 64 * 1024 }),
|
||||
function* () {
|
||||
const chunk = new Uint8Array(oneKB).fill(
|
||||
charset.LATIN_SMALL_LETTER_A,
|
||||
);
|
||||
yield quote;
|
||||
for (let index = 0; index < kbsIn200MBs; index++) {
|
||||
yield chunk;
|
||||
}
|
||||
yield quote;
|
||||
},
|
||||
({ value }) => expect((value as string).length).toEqual(twoHundredMB),
|
||||
);
|
||||
});
|
||||
|
||||
test("can handle large numbers without running out of memory", async () => {
|
||||
const jsonParser = new JSONParser({ numberBufferSize: 64 * 1024 });
|
||||
await runJSONParserTest(
|
||||
jsonParser,
|
||||
function* () {
|
||||
const chunk = new Uint8Array(oneKB).fill(charset.DIGIT_ONE);
|
||||
yield "1.";
|
||||
for (let index = 0; index < kbsIn200MBs; index++) {
|
||||
yield chunk;
|
||||
}
|
||||
},
|
||||
({ value }) => expect(value).toEqual(1.1111111111111112),
|
||||
);
|
||||
jsonParser.end();
|
||||
});
|
||||
});
|
||||
|
||||
test(`should keep memory stable if keepStack === false on array`, async () => {
|
||||
const chunk = new Uint8Array(oneKB).fill(charset.LATIN_SMALL_LETTER_A);
|
||||
chunk[0] = charset.QUOTATION_MARK;
|
||||
chunk[chunk.length - 1] = charset.QUOTATION_MARK;
|
||||
const commaChunk = new Uint8Array([charset.COMMA]);
|
||||
|
||||
const intialMemoryUsage = process.memoryUsage().heapUsed;
|
||||
const thirtyMBs = 20 * 1024 * 1024;
|
||||
let valuesLeft = kbsIn200MBs;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser({
|
||||
paths: ["$.*"],
|
||||
keepStack: false,
|
||||
stringBufferSize: oneKB,
|
||||
}),
|
||||
function* () {
|
||||
yield new Uint8Array([charset.LEFT_SQUARE_BRACKET]);
|
||||
// decreasing so the number doesn't need to be reallocated
|
||||
for (let index = kbsIn200MBs; index > 0; index--) {
|
||||
yield chunk;
|
||||
yield commaChunk;
|
||||
}
|
||||
yield chunk;
|
||||
yield new Uint8Array([charset.RIGHT_SQUARE_BRACKET]);
|
||||
},
|
||||
() => {
|
||||
if (valuesLeft-- % oneKB !== 0) return;
|
||||
|
||||
const actualMemoryUsage = process.memoryUsage().heapUsed;
|
||||
expect(actualMemoryUsage - intialMemoryUsage < thirtyMBs).toBeTruthy();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test(`should keep memory stable if keepStack === false on object`, async () => {
|
||||
const chunk = new Uint8Array(oneKB).fill(charset.LATIN_SMALL_LETTER_A);
|
||||
chunk[0] = charset.QUOTATION_MARK;
|
||||
chunk[1] = charset.LATIN_SMALL_LETTER_A;
|
||||
chunk[2] = charset.QUOTATION_MARK;
|
||||
chunk[3] = charset.COLON;
|
||||
chunk[4] = charset.QUOTATION_MARK;
|
||||
chunk[chunk.length - 1] = charset.QUOTATION_MARK;
|
||||
const commaChunk = new Uint8Array([charset.COMMA]);
|
||||
|
||||
const intialMemoryUsage = process.memoryUsage().heapUsed;
|
||||
const thirtyMBs = 20 * 1024 * 1024;
|
||||
let valuesLeft = kbsIn200MBs;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser({
|
||||
paths: ["$.*"],
|
||||
keepStack: false,
|
||||
stringBufferSize: oneKB,
|
||||
}),
|
||||
function* () {
|
||||
yield new Uint8Array([charset.LEFT_CURLY_BRACKET]);
|
||||
// decreasing so the number doesn't need to be reallocated
|
||||
for (let index = kbsIn200MBs; index > 0; index--) {
|
||||
yield chunk;
|
||||
yield commaChunk;
|
||||
}
|
||||
yield chunk;
|
||||
yield new Uint8Array([charset.RIGHT_CURLY_BRACKET]);
|
||||
},
|
||||
() => {
|
||||
if (valuesLeft-- % oneKB !== 0) return;
|
||||
|
||||
const actualMemoryUsage = process.memoryUsage().heapUsed;
|
||||
expect(actualMemoryUsage - intialMemoryUsage < thirtyMBs).toBeTruthy();
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
92
dev/env/node_modules/@streamparser/json/test/selectors.ts
generated
vendored
Executable file
92
dev/env/node_modules/@streamparser/json/test/selectors.ts
generated
vendored
Executable file
@@ -0,0 +1,92 @@
|
||||
import { runJSONParserTest, type TestData } from "./utils/testRunner.js";
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
|
||||
describe("selectors", () => {
|
||||
const testData: TestData[] = [
|
||||
{ value: "[0,1,-1]", paths: ["$"], expected: [[0, 1, -1]] },
|
||||
{ value: "[0,1,-1]", paths: ["$.*"], expected: [0, 1, -1] },
|
||||
{ value: "[0,1,-1]", expected: [0, 1, -1, [0, 1, -1]] },
|
||||
{ value: "[0,1,-1]", paths: ["$*"], expected: [0, 1, -1, [0, 1, -1]] },
|
||||
{
|
||||
value: "[0,1,[-1, 2]]",
|
||||
paths: ["$", "$.*"],
|
||||
expected: [0, 1, [-1, 2], [0, 1, [-1, 2]]],
|
||||
},
|
||||
{ value: "[0,1,-1]", paths: ["$.1"], expected: [1] },
|
||||
{
|
||||
value: '{ "a": { "b": 1, "c": 2 } }',
|
||||
paths: ["$.a.*"],
|
||||
expected: [1, 2],
|
||||
},
|
||||
{ value: '{ "a": { "b": 1, "c": 2 } }', paths: ["$.a.c"], expected: [2] },
|
||||
{
|
||||
value: '{ "a": { "b": [1,2], "c": [3, 4] } }',
|
||||
paths: ["$.a.*.*"],
|
||||
expected: [1, 2, 3, 4],
|
||||
},
|
||||
{
|
||||
value: '{ "a": { "b": [1,2], "c": [3, 4] } }',
|
||||
paths: ["$.a.*.1"],
|
||||
expected: [2, 4],
|
||||
},
|
||||
{
|
||||
value: '{ "a": { "b": [1,2], "c": [3, 4] } }',
|
||||
paths: ["$.a.c.*"],
|
||||
expected: [3, 4],
|
||||
},
|
||||
{
|
||||
value: '{ "a": { "b": [1,2], "c": [3, 4] } }',
|
||||
paths: ["$.a.c.1"],
|
||||
expected: [4],
|
||||
},
|
||||
{
|
||||
value: '{ "a": [ {"b": 1}, {"c": 2} ] }',
|
||||
paths: ["$.a.0.b"],
|
||||
expected: [1],
|
||||
},
|
||||
];
|
||||
|
||||
testData.forEach(({ value, paths, expected }) => {
|
||||
test(`Using selector ${paths} should emit only selected values`, async () => {
|
||||
let i = 0;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ paths }),
|
||||
[value],
|
||||
({ value }) => {
|
||||
expect(value).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
|
||||
expect(i).toEqual(expected.length);
|
||||
});
|
||||
});
|
||||
|
||||
const invalidTestData = [
|
||||
{
|
||||
paths: ["*"],
|
||||
expectedError: 'Invalid selector "*". Should start with "$".',
|
||||
},
|
||||
{
|
||||
paths: [".*"],
|
||||
expectedError: 'Invalid selector ".*". Should start with "$".',
|
||||
},
|
||||
{
|
||||
paths: ["$..*"],
|
||||
expectedError: 'Invalid selector "$..*". ".." syntax not supported.',
|
||||
},
|
||||
];
|
||||
|
||||
invalidTestData.forEach(({ paths, expectedError }) => {
|
||||
test(`fail on invalid selector ${paths}`, () => {
|
||||
try {
|
||||
new JSONParser({ paths });
|
||||
fail("Error expected on invalid selector");
|
||||
} catch (err: unknown) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect((err as Error).message).toEqual(expectedError);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
119
dev/env/node_modules/@streamparser/json/test/separator.ts
generated
vendored
Executable file
119
dev/env/node_modules/@streamparser/json/test/separator.ts
generated
vendored
Executable file
@@ -0,0 +1,119 @@
|
||||
import {
|
||||
runJSONParserTest,
|
||||
runTokenParserTest,
|
||||
TestData,
|
||||
} from "./utils/testRunner.js";
|
||||
import JSONParser from "../src/jsonparser.js";
|
||||
import TokenParser from "../src/tokenparser.js";
|
||||
import TokenType from "../src/utils/types/tokenType.js";
|
||||
|
||||
describe("separator", () => {
|
||||
const testData: TestData[] = [
|
||||
{ value: "true", expected: [true] },
|
||||
{ value: "false", expected: [false] },
|
||||
{ value: "null", expected: [null] },
|
||||
{ value: '"string"', expected: ["string"] },
|
||||
{ value: "[1,2,3]", expected: [1, 2, 3, [1, 2, 3]] },
|
||||
{
|
||||
value: '{ "a": 0, "b": 1, "c": -1 }',
|
||||
expected: [0, 1, -1, { a: 0, b: 1, c: -1 }],
|
||||
},
|
||||
];
|
||||
|
||||
const expected = testData
|
||||
.map(({ expected }) => expected)
|
||||
.reduce((acc, val) => [...acc, ...val], []);
|
||||
|
||||
const separators = ["", "\n", "\t\n", "abc", "SEPARATOR"];
|
||||
separators.forEach((separator) => {
|
||||
test(`separator: "${separator}"`, async () => {
|
||||
let i = 0;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ separator }),
|
||||
testData.flatMap(({ value }) => [value, separator]),
|
||||
({ value }) => {
|
||||
expect(value).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("support multiple whitespace separators", async () => {
|
||||
let i = 0;
|
||||
const value = "1 2\t3\n4\n\r5 \n6\n\n7\n\r\n\r8 \t\n\n\r9";
|
||||
const expected = [1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
const separator = "";
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ separator }),
|
||||
value,
|
||||
({ value }) => {
|
||||
expect(value).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test(`separator: fail on invalid value`, async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser({ separator: "abc" }), ["abe"]);
|
||||
} catch (err: unknown) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect((err as Error).message).toEqual(
|
||||
'Unexpected "e" at position "2" in state SEPARATOR',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test(`fail on invalid token type`, async () => {
|
||||
try {
|
||||
await runTokenParserTest(new TokenParser({ separator: "\n" }), [
|
||||
{ token: TokenType.TRUE, value: true },
|
||||
{ token: TokenType.TRUE, value: true },
|
||||
]);
|
||||
fail("Error expected on invalid selector");
|
||||
} catch (err: unknown) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect((err as Error).message).toEqual(
|
||||
"Unexpected TRUE (true) in state SEPARATOR",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("fail on invalid value passed to TokenParser", async () => {
|
||||
try {
|
||||
await runTokenParserTest(new TokenParser({ separator: "\n" }), [
|
||||
{ token: TokenType.TRUE, value: true },
|
||||
{ token: TokenType.SEPARATOR, value: "\r\n" },
|
||||
]);
|
||||
fail("Error expected on invalid selector");
|
||||
} catch (err: unknown) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect((err as Error).message).toEqual(
|
||||
'Unexpected SEPARATOR ("\\r\\n") in state SEPARATOR',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("not fail when whitespaces match separator", async () => {
|
||||
let i = 0;
|
||||
const value = `{
|
||||
"a": 0,
|
||||
"b": 1,
|
||||
"c": -1
|
||||
}`;
|
||||
const expected = [0, 1, -1, { a: 0, b: 1, c: -1 }];
|
||||
const separator = "\n";
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ separator }),
|
||||
value,
|
||||
({ value }) => {
|
||||
expect(value).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
109
dev/env/node_modules/@streamparser/json/test/types/arrays.ts
generated
vendored
Executable file
109
dev/env/node_modules/@streamparser/json/test/types/arrays.ts
generated
vendored
Executable file
@@ -0,0 +1,109 @@
|
||||
import { runJSONParserTest, type TestData } from "../utils/testRunner.js";
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
|
||||
describe("arrays", () => {
|
||||
const testData: TestData[] = [
|
||||
{ value: "[]", expected: [[[], []]] },
|
||||
{
|
||||
value: "[0,1,-1]",
|
||||
expected: [
|
||||
[[0], 0],
|
||||
[[1], 1],
|
||||
[[2], -1],
|
||||
[[], [0, 1, -1]],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: "[1.0,1.1,-1.1,-1.0]",
|
||||
expected: [
|
||||
[[0], 1],
|
||||
[[1], 1.1],
|
||||
[[2], -1.1],
|
||||
[[3], -1],
|
||||
[[], [1, 1.1, -1.1, -1]],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: "[-1]",
|
||||
expected: [
|
||||
[[0], -1],
|
||||
[[], [-1]],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: "[-0.1]",
|
||||
expected: [
|
||||
[[0], -0.1],
|
||||
[[], [-0.1]],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: "[6.02e23, 6.02e+23, 6.02e-23, 0e23]",
|
||||
expected: [
|
||||
[[0], 6.02e23],
|
||||
[[1], 6.02e23],
|
||||
[[2], 6.02e-23],
|
||||
[[3], 0e23],
|
||||
[[], [6.02e23, 6.02e23, 6.02e-23, 0e23]],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: "[7161093205057351174]",
|
||||
expected: [
|
||||
[[0], 7161093205057352000],
|
||||
[[], [7161093205057352000]],
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
testData.forEach(({ value, expected }) => {
|
||||
test(value as string, async () => {
|
||||
let i = 0;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
[value],
|
||||
({ value, key, stack }) => {
|
||||
const keys = stack
|
||||
.slice(1)
|
||||
.map((item) => item.key)
|
||||
.concat(key !== undefined ? key : []);
|
||||
|
||||
expect([keys, value]).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("chuncked", async () => {
|
||||
let i = 0;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
(value as string).split(""),
|
||||
({ value, key, stack }) => {
|
||||
const keys = stack
|
||||
.slice(1)
|
||||
.map((item) => item.key)
|
||||
.concat(key !== undefined ? key : []);
|
||||
|
||||
expect([keys, value]).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const invalidValues = ["[,", "[1, eer]", "[1,]", "[1;", "[1}"];
|
||||
|
||||
invalidValues.forEach((value) => {
|
||||
test(`fail on invalid values: ${value}`, async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser(), [value]);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
45
dev/env/node_modules/@streamparser/json/test/types/booleans.ts
generated
vendored
Executable file
45
dev/env/node_modules/@streamparser/json/test/types/booleans.ts
generated
vendored
Executable file
@@ -0,0 +1,45 @@
|
||||
import { runJSONParserTest } from "../utils/testRunner.js";
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
|
||||
describe("boolean", () => {
|
||||
const values = ["true", "false"];
|
||||
|
||||
values.forEach((stringValue) => {
|
||||
test(stringValue, async () => {
|
||||
await runJSONParserTest(new JSONParser(), [stringValue], ({ value }) => {
|
||||
expect(value).toEqual(JSON.parse(stringValue));
|
||||
});
|
||||
});
|
||||
|
||||
test(`${stringValue} (chuncked)`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
(stringValue as string).split(""),
|
||||
({ value }) => {
|
||||
expect(value).toEqual(JSON.parse(stringValue));
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const invalidValues = [
|
||||
"tRue",
|
||||
"trUe",
|
||||
"truE",
|
||||
"fAlse",
|
||||
"faLse",
|
||||
"falSe",
|
||||
"falsE",
|
||||
];
|
||||
|
||||
invalidValues.forEach((value) => {
|
||||
test("fail on invalid values", async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser(), [value]);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
37
dev/env/node_modules/@streamparser/json/test/types/null.ts
generated
vendored
Executable file
37
dev/env/node_modules/@streamparser/json/test/types/null.ts
generated
vendored
Executable file
@@ -0,0 +1,37 @@
|
||||
import { runJSONParserTest } from "../utils/testRunner.js";
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
|
||||
describe("null", () => {
|
||||
const values = ["null"];
|
||||
|
||||
values.forEach((stringValue) => {
|
||||
test(stringValue, async () => {
|
||||
await runJSONParserTest(new JSONParser(), [stringValue], ({ value }) => {
|
||||
expect(value).toEqual(JSON.parse(stringValue));
|
||||
});
|
||||
});
|
||||
|
||||
test(`${stringValue} (chuncked)`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
(stringValue as string).split(""),
|
||||
({ value }) => {
|
||||
expect(value).toEqual(JSON.parse(stringValue));
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const invalidValues = ["nUll", "nuLl", "nulL"];
|
||||
|
||||
invalidValues.forEach((value) => {
|
||||
test("fail on invalid values", async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser(), [value]);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
101
dev/env/node_modules/@streamparser/json/test/types/numbers.ts
generated
vendored
Executable file
101
dev/env/node_modules/@streamparser/json/test/types/numbers.ts
generated
vendored
Executable file
@@ -0,0 +1,101 @@
|
||||
import { runJSONParserTest } from "../utils/testRunner.js";
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
|
||||
describe("number", () => {
|
||||
const values = [
|
||||
"0",
|
||||
"0e1",
|
||||
"0e+1",
|
||||
"0e-1",
|
||||
"0.123",
|
||||
"0.123e00",
|
||||
"0.123e+1",
|
||||
"0.123e-1",
|
||||
"0.123E00",
|
||||
"0.123E+1",
|
||||
"0.123E-1",
|
||||
"-0",
|
||||
"-0e1",
|
||||
"-0e+1",
|
||||
"-0e-1",
|
||||
"-0.123",
|
||||
"-0.123e00",
|
||||
"-0.123e+1",
|
||||
"-0.123e-1",
|
||||
"-0.123E00",
|
||||
"-0.123E+1",
|
||||
"-0.123E-1",
|
||||
"-123",
|
||||
"-123e1",
|
||||
"-123e+1",
|
||||
"-123e-1",
|
||||
"-123.123",
|
||||
"-123.123e00",
|
||||
"-123.123e+1",
|
||||
"-123.123e-1",
|
||||
"-123.123E00",
|
||||
"-123.123E+1",
|
||||
"-123.123E-1",
|
||||
"123",
|
||||
"123e1",
|
||||
"123e+1",
|
||||
"123e-1",
|
||||
"123.123",
|
||||
"123.123e00",
|
||||
"123.123e+1",
|
||||
"123.123e-1",
|
||||
"123.123E00",
|
||||
"123.123E+1",
|
||||
"123.123E-1",
|
||||
"7161093205057351174",
|
||||
"21e999",
|
||||
];
|
||||
|
||||
const bufferSizes = [0, 1, 64 * 1024];
|
||||
|
||||
bufferSizes.forEach((numberBufferSize) => {
|
||||
values.forEach((stringValue) => {
|
||||
test(`${stringValue} (bufferSize ${numberBufferSize})`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ numberBufferSize }),
|
||||
[stringValue],
|
||||
({ value }) => {
|
||||
expect(value).toEqual(JSON.parse(stringValue));
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test(`${stringValue} (chunked, bufferSize ${numberBufferSize})`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ numberBufferSize }),
|
||||
(stringValue as string).split(""),
|
||||
({ value }) => {
|
||||
expect(value).toEqual(JSON.parse(stringValue));
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const invalidValues = [
|
||||
"-a",
|
||||
"-e",
|
||||
"1a",
|
||||
"1.a",
|
||||
"1.e",
|
||||
"1.-",
|
||||
"1.0ea",
|
||||
"1.0e1.2",
|
||||
];
|
||||
|
||||
invalidValues.forEach((value) => {
|
||||
test("fail on invalid values", async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser(), [value]);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
134
dev/env/node_modules/@streamparser/json/test/types/objects.ts
generated
vendored
Executable file
134
dev/env/node_modules/@streamparser/json/test/types/objects.ts
generated
vendored
Executable file
@@ -0,0 +1,134 @@
|
||||
import { runJSONParserTest, type TestData } from "../utils/testRunner.js";
|
||||
import { readFileSync } from "fs";
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
|
||||
describe("objects", () => {
|
||||
const testData: TestData[] = [
|
||||
{ value: "{}", expected: [[[], {}]] },
|
||||
{
|
||||
value: '{ "a": 0, "b": 1, "c": -1 }',
|
||||
expected: [
|
||||
[["a"], 0],
|
||||
[["b"], 1],
|
||||
[["c"], -1],
|
||||
[[], { a: 0, b: 1, c: -1 }],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "a": 1.0, "b": 1.1, "c": -1.1, "d": -1.0 }',
|
||||
expected: [
|
||||
[["a"], 1],
|
||||
[["b"], 1.1],
|
||||
[["c"], -1.1],
|
||||
[["d"], -1],
|
||||
[[], { a: 1, b: 1.1, c: -1.1, d: -1 }],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "e": -1 }',
|
||||
expected: [
|
||||
[["e"], -1],
|
||||
[[], { e: -1 }],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "f": -0.1 }',
|
||||
expected: [
|
||||
[["f"], -0.1],
|
||||
[[], { f: -0.1 }],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "a": 6.02e23, "b": 6.02e+23, "c": 6.02e-23, "d": 0e23 }',
|
||||
expected: [
|
||||
[["a"], 6.02e23],
|
||||
[["b"], 6.02e23],
|
||||
[["c"], 6.02e-23],
|
||||
[["d"], 0e23],
|
||||
[[], { a: 6.02e23, b: 6.02e23, c: 6.02e-23, d: 0e23 }],
|
||||
],
|
||||
},
|
||||
{
|
||||
value: '{ "a": 7161093205057351174 }',
|
||||
expected: [
|
||||
[["a"], 7161093205057352000],
|
||||
[[], { a: 7161093205057352000 }],
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
testData.forEach(({ value, expected }) => {
|
||||
test(value as string, async () => {
|
||||
let i = 0;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
[value],
|
||||
({ value, key, stack }) => {
|
||||
const keys = stack
|
||||
.slice(1)
|
||||
.map((item) => item.key)
|
||||
.concat(key !== undefined ? key : []);
|
||||
|
||||
expect([keys, value]).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("chuncked", async () => {
|
||||
let i = 0;
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
(value as string).split(""),
|
||||
({ value, key, stack }) => {
|
||||
const keys = stack
|
||||
.slice(1)
|
||||
.map((item) => item.key)
|
||||
.concat(key !== undefined ? key : []);
|
||||
|
||||
expect([keys, value]).toEqual(expected[i]);
|
||||
i += 1;
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("complex object", async () => {
|
||||
const stringifiedJson = readFileSync(
|
||||
`${__dirname}/../../../../samplejson/basic.json`,
|
||||
).toString();
|
||||
|
||||
await runJSONParserTest(
|
||||
new JSONParser(),
|
||||
[stringifiedJson],
|
||||
({ value, stack }) => {
|
||||
if (stack.length === 0) {
|
||||
expect(value).toEqual(JSON.parse(stringifiedJson));
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const invalidValues = [
|
||||
"{,",
|
||||
'{"test": eer[ }',
|
||||
"{ test: 1 }",
|
||||
'{ "test": 1 ;',
|
||||
'{ "test": 1 ]',
|
||||
'{ "test": 1, }',
|
||||
'{ "test", }',
|
||||
];
|
||||
|
||||
invalidValues.forEach((value) => {
|
||||
test(`fail on invalid values: ${value}`, async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser(), [value]);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
184
dev/env/node_modules/@streamparser/json/test/types/strings.ts
generated
vendored
Executable file
184
dev/env/node_modules/@streamparser/json/test/types/strings.ts
generated
vendored
Executable file
@@ -0,0 +1,184 @@
|
||||
import { runJSONParserTest } from "../utils/testRunner.js";
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
import { charset } from "../../src/utils/utf-8.js";
|
||||
|
||||
const quote = String.fromCharCode(charset.QUOTATION_MARK);
|
||||
|
||||
describe("string", () => {
|
||||
const values = [
|
||||
"Hello world!",
|
||||
'\\r\\n\\f\\t\\\\\\/\\"',
|
||||
"\\u039b\\u03ac\\u03bc\\u03b2\\u03b4\\u03b1",
|
||||
"☃",
|
||||
"├──",
|
||||
"snow: ☃!",
|
||||
"õ",
|
||||
];
|
||||
|
||||
const bufferSizes = [0, 1, 64 * 1024];
|
||||
|
||||
bufferSizes.forEach((stringBufferSize) => {
|
||||
values.forEach((stringValue) => {
|
||||
test(`${stringValue} (bufferSize ${stringBufferSize})`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, stringValue, quote],
|
||||
({ value }) => expect(value).toEqual(JSON.parse(`"${stringValue}"`)),
|
||||
);
|
||||
});
|
||||
|
||||
test(`${stringValue} (chunked, bufferSize ${stringBufferSize})`, async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, ...(stringValue as string).split(""), quote],
|
||||
({ value }) => expect(value).toEqual(JSON.parse(`"${stringValue}"`)),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("multibyte characters", () => {
|
||||
test("2 byte utf8 'De' character: д", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, new Uint8Array([0xd0, 0xb4]), quote],
|
||||
({ value }) => expect(value).toEqual("д"),
|
||||
);
|
||||
});
|
||||
|
||||
test("3 byte utf8 'Han' character: 我", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, new Uint8Array([0xe6, 0x88, 0x91]), quote],
|
||||
({ value }) => expect(value).toEqual("我"),
|
||||
);
|
||||
});
|
||||
|
||||
test("4 byte utf8 character (unicode scalar U+2070E): 𠜎", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, new Uint8Array([0xf0, 0xa0, 0x9c, 0x8e]), quote],
|
||||
({ value }) => expect(value).toEqual("𠜎"),
|
||||
);
|
||||
});
|
||||
|
||||
describe("chunking", () => {
|
||||
test("2 byte utf8 'De' character chunked inbetween 1st and 3nd byte: д", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, new Uint8Array([0xd0]), new Uint8Array([0xb4]), quote],
|
||||
({ value }) => expect(value).toEqual("д"),
|
||||
);
|
||||
});
|
||||
|
||||
test("3 byte utf8 'Han' character chunked inbetween 2nd and 3rd byte: 我", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[
|
||||
quote,
|
||||
new Uint8Array([0xe6, 0x88]),
|
||||
new Uint8Array([0x91]),
|
||||
quote,
|
||||
],
|
||||
({ value }) => expect(value).toEqual("我"),
|
||||
);
|
||||
});
|
||||
|
||||
test("4 byte utf8 character (unicode scalar U+2070E) chunked inbetween 2nd and 3rd byte: 𠜎", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[
|
||||
quote,
|
||||
new Uint8Array([0xf0, 0xa0]),
|
||||
new Uint8Array([0x9c, 0x8e]),
|
||||
quote,
|
||||
],
|
||||
({ value }) => expect(value).toEqual("𠜎"),
|
||||
);
|
||||
});
|
||||
|
||||
test("1-4 byte utf8 character string chunked inbetween random bytes: Aж文𠜱B", async () => {
|
||||
const eclectic_buffer = new Uint8Array([
|
||||
0x41, // A
|
||||
0xd0,
|
||||
0xb6, // ж
|
||||
0xe6,
|
||||
0x96,
|
||||
0x87, // 文
|
||||
0xf0,
|
||||
0xa0,
|
||||
0x9c,
|
||||
0xb1, // 𠜱
|
||||
0x42,
|
||||
]); // B
|
||||
|
||||
for (let i = 0; i < 11; i++) {
|
||||
const firstBuffer = eclectic_buffer.slice(0, i);
|
||||
const secondBuffer = eclectic_buffer.slice(i);
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, firstBuffer, secondBuffer, quote],
|
||||
({ value }) => expect(value).toEqual("Aж文𠜱B"),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("surrogate", () => {
|
||||
test("parse surrogate pair", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, "\\uD83D\\uDE0B", quote],
|
||||
({ value }) => expect(value).toEqual("😋"),
|
||||
);
|
||||
});
|
||||
|
||||
test("surrogate pair (chunked)", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, "\\uD83D", "\\uDE0B", quote],
|
||||
({ value }) => expect(value).toEqual("😋"),
|
||||
);
|
||||
});
|
||||
|
||||
test("not error on broken surrogate pair", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize }),
|
||||
[quote, "\\uD83D\\uEFFF", quote],
|
||||
({ value }) => expect(value).toEqual("<22>"),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("should flush the buffer if there is not space for incoming data", async () => {
|
||||
await runJSONParserTest(
|
||||
new JSONParser({ stringBufferSize: 1 }),
|
||||
[quote, "aaaa", "𠜎", quote],
|
||||
({ value }) => expect(value).toEqual("aaaa𠜎"),
|
||||
);
|
||||
});
|
||||
|
||||
const invalidValues = [
|
||||
'"\n"',
|
||||
'"\\j"',
|
||||
'"\\ua"',
|
||||
'"\\u1*"',
|
||||
'"\\u12*"',
|
||||
"\\u123*",
|
||||
'"\0"',
|
||||
'"\\uG"',
|
||||
'"\\u000G"',
|
||||
];
|
||||
|
||||
invalidValues.forEach((value) => {
|
||||
test(`fail on invalid values ${value}`, async () => {
|
||||
try {
|
||||
await runJSONParserTest(new JSONParser(), value);
|
||||
fail(`Expected to fail on value "${value}"`);
|
||||
} catch {
|
||||
// Expected error
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
62
dev/env/node_modules/@streamparser/json/test/utils/testRunner.ts
generated
vendored
Executable file
62
dev/env/node_modules/@streamparser/json/test/utils/testRunner.ts
generated
vendored
Executable file
@@ -0,0 +1,62 @@
|
||||
import JSONParser from "../../src/jsonparser.js";
|
||||
import Tokenizer from "../../src/tokenizer.js";
|
||||
import TokenParser from "../../src/tokenparser.js";
|
||||
import { ParsedTokenInfo } from "../../src/utils/types/parsedTokenInfo.js";
|
||||
import { ParsedElementInfo } from "../../src/utils/types/parsedElementInfo.js";
|
||||
|
||||
export type TestData = {
|
||||
value: string | string[] | Iterable<number>;
|
||||
paths?: string[];
|
||||
expected: unknown[];
|
||||
};
|
||||
|
||||
type ParseableData = string | Iterable<number>;
|
||||
type InputData<T> = T | T[] | (() => Generator<T>);
|
||||
|
||||
function iterableData<T>(data: InputData<T>): Iterable<T> {
|
||||
if (typeof data === "function") return (data as () => Generator<T>)();
|
||||
if (Array.isArray(data)) return data;
|
||||
return [data];
|
||||
}
|
||||
|
||||
export function runJSONParserTest(
|
||||
jsonparser: JSONParser,
|
||||
data: InputData<ParseableData>,
|
||||
onValue: (parsedElementInfo: ParsedElementInfo) => void = () => {
|
||||
/* Do nothing */
|
||||
},
|
||||
) {
|
||||
jsonparser.onValue = onValue;
|
||||
for (const value of iterableData(data)) {
|
||||
jsonparser.write(value);
|
||||
}
|
||||
// jsonparser.end();
|
||||
}
|
||||
|
||||
export function runTokenizerTest(
|
||||
tokenizer: Tokenizer,
|
||||
data: InputData<ParseableData>,
|
||||
onToken: (parsedElementInfo: ParsedTokenInfo) => void = () => {
|
||||
/* Do nothing */
|
||||
},
|
||||
) {
|
||||
tokenizer.onToken = onToken;
|
||||
for (const value of iterableData(data)) {
|
||||
tokenizer.write(value);
|
||||
}
|
||||
tokenizer.end();
|
||||
}
|
||||
|
||||
export function runTokenParserTest(
|
||||
tokenParser: TokenParser,
|
||||
data: InputData<Omit<ParsedTokenInfo, "offset">>,
|
||||
onValue: (parsedElementInfo: ParsedElementInfo) => void = () => {
|
||||
/* Do nothing */
|
||||
},
|
||||
) {
|
||||
tokenParser.onValue = onValue;
|
||||
for (const value of iterableData(data)) {
|
||||
tokenParser.write(value);
|
||||
}
|
||||
tokenParser.end();
|
||||
}
|
||||
Reference in New Issue
Block a user