From e4fe80d003fdc199ff1134654f81b27496331da6 Mon Sep 17 00:00:00 2001 From: Alive24 Date: Sat, 17 May 2025 14:12:01 +0100 Subject: [PATCH 1/5] feat: compatible mode for molecule decode --- packages/core/src/molecule/codec.ts | 65 +++++++++++++++++------------ 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/packages/core/src/molecule/codec.ts b/packages/core/src/molecule/codec.ts index 734b13f1..324d53a2 100644 --- a/packages/core/src/molecule/codec.ts +++ b/packages/core/src/molecule/codec.ts @@ -18,13 +18,19 @@ import { export type CodecLike = { readonly encode: (encodable: Encodable) => Bytes; - readonly decode: (decodable: BytesLike) => Decoded; + readonly decode: ( + decodable: BytesLike, + config?: { compatible?: boolean }, + ) => Decoded; readonly byteLength?: number; }; export class Codec { constructor( public readonly encode: (encodable: Encodable) => Bytes, - public readonly decode: (decodable: BytesLike) => Decoded, + public readonly decode: ( + decodable: BytesLike, + config?: { compatible?: boolean }, + ) => Decoded, public readonly byteLength?: number, // if provided, treat codec as fixed length ) {} @@ -43,7 +49,7 @@ export class Codec { } return encoded; }, - (decodable) => { + (decodable, config = { compatible: false }) => { const decodableBytes = bytesFrom(decodable); if ( byteLength !== undefined && @@ -53,7 +59,7 @@ export class Codec { `Codec.decode: expected byte length ${byteLength}, got ${decodableBytes.byteLength}`, ); } - return decode(decodable); + return decode(decodable, config); }, byteLength, ); @@ -69,10 +75,10 @@ export class Codec { return new Codec( (encodable) => this.encode((inMap ? inMap(encodable) : encodable) as Encodable), - (buffer) => + (buffer, config = { compatible: false }) => (outMap - ? outMap(this.decode(buffer)) - : this.decode(buffer)) as NewDecoded, + ? outMap(this.decode(buffer, config)) + : this.decode(buffer, config)) as NewDecoded, this.byteLength, ); } @@ -128,7 +134,7 @@ export function fixedItemVec( throw new Error(`fixedItemVec(${e?.toString()})`); } }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); if (value.byteLength < 4) { throw new Error( @@ -147,7 +153,10 @@ export function fixedItemVec( const decodedArray: Array = []; for (let offset = 4; offset < byteLength; offset += itemByteLength) { decodedArray.push( - itemCodec.decode(value.slice(offset, offset + itemByteLength)), + itemCodec.decode( + value.slice(offset, offset + itemByteLength), + config, + ), ); } return decodedArray; @@ -185,7 +194,7 @@ export function dynItemVec( throw new Error(`dynItemVec(${e?.toString()})`); } }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); if (value.byteLength < 4) { throw new Error( @@ -215,7 +224,7 @@ export function dynItemVec( const start = offsets[index]; const end = offsets[index + 1]; const itemBuffer = value.slice(start, end); - decodedArray.push(itemCodec.decode(itemBuffer)); + decodedArray.push(itemCodec.decode(itemBuffer, config)); } return decodedArray; } catch (e) { @@ -259,13 +268,13 @@ export function option( throw new Error(`option(${e?.toString()})`); } }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); if (value.byteLength === 0) { return undefined; } try { - return innerCodec.decode(buffer); + return innerCodec.decode(buffer, config); } catch (e) { throw new Error(`option(${e?.toString()})`); } @@ -290,7 +299,7 @@ export function byteVec( throw new Error(`byteVec(${e?.toString()})`); } }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); if (value.byteLength < 4) { throw new Error( @@ -304,7 +313,7 @@ export function byteVec( ); } try { - return codec.decode(value.slice(4)); + return codec.decode(value.slice(4), config); } catch (e: unknown) { throw new Error(`byteVec(${e?.toString()})`); } @@ -371,7 +380,7 @@ export function table< const packedTotalSize = uint32To(header.length + body.length + 4); return bytesConcat(packedTotalSize, header, body); }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); if (value.byteLength < 4) { throw new Error( @@ -379,7 +388,7 @@ export function table< ); } const byteLength = uint32From(value.slice(0, 4)); - if (byteLength !== value.byteLength) { + if (byteLength !== value.byteLength && !config?.compatible) { throw new Error( `table: invalid buffer size, expected ${byteLength}, but got ${value.byteLength}`, ); @@ -397,9 +406,13 @@ export function table< const payload = value.slice(start, end); try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - Object.assign(object, { [field]: codec.decode(payload) }); - } catch (e: unknown) { - throw new Error(`table.${field}(${e?.toString()})`); + Object.assign(object, { [field]: codec.decode(payload, config) }); + } catch (_e: unknown) { + if (config?.compatible) { + Object.assign(object, { [field]: null }); + } else { + throw new Error(`table.${field}(${_e?.toString()})`); + } } } return object as Decoded; @@ -466,7 +479,7 @@ export function union>>( throw new Error(`union.(${typeStr})(${e?.toString()})`); } }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); const fieldIndex = uint32From(value.slice(0, 4)); const keys = Object.keys(codecLayout); @@ -496,7 +509,7 @@ export function union>>( return { type: field, // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - value: codecLayout[field].decode(value.slice(4)), + value: codecLayout[field].decode(value.slice(4), config), } as UnionDecoded; }, }); @@ -535,7 +548,7 @@ export function struct< return bytesFrom(bytes); }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); const object = {}; let offset = 0; @@ -543,7 +556,7 @@ export function struct< const payload = value.slice(offset, offset + codec.byteLength!); try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - Object.assign(object, { [key]: codec.decode(payload) }); + Object.assign(object, { [key]: codec.decode(payload, config) }); } catch (e: unknown) { throw new Error(`struct.${key}(${(e as Error).toString()})`); } @@ -583,7 +596,7 @@ export function array( throw new Error(`array(${e?.toString()})`); } }, - decode(buffer) { + decode(buffer, config) { const value = bytesFrom(buffer); if (value.byteLength != byteLength) { throw new Error( @@ -594,7 +607,7 @@ export function array( const result: Array = []; for (let i = 0; i < value.byteLength; i += itemCodec.byteLength!) { result.push( - itemCodec.decode(value.slice(i, i + itemCodec.byteLength!)), + itemCodec.decode(value.slice(i, i + itemCodec.byteLength!), config), ); } return result; From 9dab7b0b15ad05c7070a003e49aac5b3ba138512 Mon Sep 17 00:00:00 2001 From: Alive24 Date: Sat, 17 May 2025 15:33:29 +0000 Subject: [PATCH 2/5] fix: minor fixes --- packages/core/src/molecule/codec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/src/molecule/codec.ts b/packages/core/src/molecule/codec.ts index 324d53a2..0b80695f 100644 --- a/packages/core/src/molecule/codec.ts +++ b/packages/core/src/molecule/codec.ts @@ -20,7 +20,7 @@ export type CodecLike = { readonly encode: (encodable: Encodable) => Bytes; readonly decode: ( decodable: BytesLike, - config?: { compatible?: boolean }, + config?: { isExtraFieldIgnored?: boolean }, ) => Decoded; readonly byteLength?: number; }; @@ -29,7 +29,7 @@ export class Codec { public readonly encode: (encodable: Encodable) => Bytes, public readonly decode: ( decodable: BytesLike, - config?: { compatible?: boolean }, + config?: { isExtraFieldIgnored?: boolean }, ) => Decoded, public readonly byteLength?: number, // if provided, treat codec as fixed length ) {} @@ -49,7 +49,7 @@ export class Codec { } return encoded; }, - (decodable, config = { compatible: false }) => { + (decodable, config = { isExtraFieldIgnored: false }) => { const decodableBytes = bytesFrom(decodable); if ( byteLength !== undefined && @@ -75,7 +75,7 @@ export class Codec { return new Codec( (encodable) => this.encode((inMap ? inMap(encodable) : encodable) as Encodable), - (buffer, config = { compatible: false }) => + (buffer, config = { isExtraFieldIgnored: false }) => (outMap ? outMap(this.decode(buffer, config)) : this.decode(buffer, config)) as NewDecoded, @@ -388,7 +388,7 @@ export function table< ); } const byteLength = uint32From(value.slice(0, 4)); - if (byteLength !== value.byteLength && !config?.compatible) { + if (byteLength !== value.byteLength) { throw new Error( `table: invalid buffer size, expected ${byteLength}, but got ${value.byteLength}`, ); @@ -408,7 +408,7 @@ export function table< // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment Object.assign(object, { [field]: codec.decode(payload, config) }); } catch (_e: unknown) { - if (config?.compatible) { + if (config?.isExtraFieldIgnored) { Object.assign(object, { [field]: null }); } else { throw new Error(`table.${field}(${_e?.toString()})`); From d66a1fb540c7194200c47607cee8310d3ec4cdf6 Mon Sep 17 00:00:00 2001 From: Alive24 Date: Sat, 17 May 2025 15:35:29 +0000 Subject: [PATCH 3/5] doc: minor comment --- packages/core/src/molecule/codec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/molecule/codec.ts b/packages/core/src/molecule/codec.ts index 0b80695f..6b500bc6 100644 --- a/packages/core/src/molecule/codec.ts +++ b/packages/core/src/molecule/codec.ts @@ -29,7 +29,7 @@ export class Codec { public readonly encode: (encodable: Encodable) => Bytes, public readonly decode: ( decodable: BytesLike, - config?: { isExtraFieldIgnored?: boolean }, + config?: { isExtraFieldIgnored?: boolean }, // This is equivalent to "compatible" in the Rust implementation of Molecule. ) => Decoded, public readonly byteLength?: number, // if provided, treat codec as fixed length ) {} From e570623b54d04db1cc515a06f6d8b0a0399a54f0 Mon Sep 17 00:00:00 2001 From: Alive24 Date: Mon, 19 May 2025 22:47:37 +0100 Subject: [PATCH 4/5] fix: improved implementation of compatibility mode based on #209 --- packages/core/src/molecule/codec.ts | 37 ++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/core/src/molecule/codec.ts b/packages/core/src/molecule/codec.ts index 6b500bc6..cf0af43a 100644 --- a/packages/core/src/molecule/codec.ts +++ b/packages/core/src/molecule/codec.ts @@ -49,7 +49,7 @@ export class Codec { } return encoded; }, - (decodable, config = { isExtraFieldIgnored: false }) => { + (decodable, config) => { const decodableBytes = bytesFrom(decodable); if ( byteLength !== undefined && @@ -75,7 +75,7 @@ export class Codec { return new Codec( (encodable) => this.encode((inMap ? inMap(encodable) : encodable) as Encodable), - (buffer, config = { isExtraFieldIgnored: false }) => + (buffer, config) => (outMap ? outMap(this.decode(buffer, config)) : this.decode(buffer, config)) as NewDecoded, @@ -388,15 +388,38 @@ export function table< ); } const byteLength = uint32From(value.slice(0, 4)); + const headerLength = uint32From(value.slice(4, 8)); + const actualFieldCount = (headerLength - 4) / 4; + if (byteLength !== value.byteLength) { throw new Error( `table: invalid buffer size, expected ${byteLength}, but got ${value.byteLength}`, ); } + + if (actualFieldCount < keys.length) { + throw new Error( + `table: invalid field count, expected ${keys.length}, but got ${actualFieldCount}`, + ); + } + + if (actualFieldCount > keys.length && !config?.isExtraFieldIgnored) { + throw new Error( + `table: invalid field count, expected ${keys.length}, but got ${actualFieldCount}, and extra fields are not allowed in the current configuration. If you want to ignore extra fields, set isExtraFieldIgnored to true.`, + ); + } const offsets = keys.map((_, index) => uint32From(value.slice(4 + index * 4, 8 + index * 4)), ); - offsets.push(byteLength); + // If there are extra fields, add the last offset to the offsets array + if (actualFieldCount > keys.length) { + offsets.push( + uint32From(value.slice(4 + keys.length * 4, 8 + keys.length * 4)), + ); + } else { + // If there are no extra fields, add the byte length to the offsets array + offsets.push(byteLength); + } const object = {}; for (let i = 0; i < offsets.length - 1; i++) { const start = offsets[i]; @@ -407,12 +430,8 @@ export function table< try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment Object.assign(object, { [field]: codec.decode(payload, config) }); - } catch (_e: unknown) { - if (config?.isExtraFieldIgnored) { - Object.assign(object, { [field]: null }); - } else { - throw new Error(`table.${field}(${_e?.toString()})`); - } + } catch (e: unknown) { + throw new Error(`table.${field}(${e?.toString()})`); } } return object as Decoded; From 3a5b9bee5e064f76b08fa766713cb90194d8fa18 Mon Sep 17 00:00:00 2001 From: Hanssen Date: Tue, 20 May 2025 06:35:52 +0800 Subject: [PATCH 5/5] Create olive-dryers-live.md --- .changeset/olive-dryers-live.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/olive-dryers-live.md diff --git a/.changeset/olive-dryers-live.md b/.changeset/olive-dryers-live.md new file mode 100644 index 00000000..59272911 --- /dev/null +++ b/.changeset/olive-dryers-live.md @@ -0,0 +1,5 @@ +--- +"@ckb-ccc/core": minor +--- + +feat: compatible mode for molecule decode