core-utils: update batch serialization

Update the batch serialization to allow for typed batches.
Also include logic for type 0 batches, which are compressed
with zlib.

.changeset/thin-trees-matter.md

+---
+'@eth-optimism/core-utils': patch
+---
+
+Update batch serialization with typed batches and zlib compression

packages/core-utils/package.json

@@ -35,7 +35,9 @@
     "@ethersproject/abstract-provider": "^5.5.1",
     "@ethersproject/bytes": "^5.5.0",
     "@ethersproject/providers": "^5.5.3",
+    "@ethersproject/transactions": "^5.5.0",
     "@ethersproject/web": "^5.5.1",
+    "bufio": "^1.0.7",
     "chai": "^4.3.4",
     "ethers": "^5.5.4"
   },

packages/core-utils/src/@types/external/bufio/index.d.ts

+declare module 'bufio' {
+  class BufferWriter {
+    public offset: number
+
+    constructor()
+    render(): Buffer
+    getSize(): number
+    seek(offset: number): this
+    destroy(): this
+    writeU8(n: number): this
+    writeU16(n: number): this
+    writeU16BE(n: number): this
+    writeU24(n: number): this
+    writeU24BE(n: number): this
+    writeU32(n: number): this
+    writeU32BE(n: number): this
+    writeU40(n: number): this
+    writeU40BE(n: number): this
+    writeU48(n: number): this
+    writeU48BE(n: number): this
+    writeU56(n: number): this
+    writeU56BE(n: number): this
+    writeU64(n: number): this
+    writeU64BE(n: number): this
+    writeBytes(b: Buffer): this
+    copy(value: number, start: number, end: number): this
+  }
+
+  class BufferReader {
+    constructor(data: Buffer, copy?: boolean)
+    getSize(): number
+    check(n: number): void
+    left(): number
+    seek(offset: number): this
+    start(): number
+    end(): number
+    destroy(): this
+    readU8(): number
+    readU16(): number
+    readU16BE(): number
+    readU24(): number
+    readU24BE(): number
+    readU32(): number
+    readU32BE(): number
+    readU40(): number
+    readU40BE(): number
+    readU48(): number
+    readU48BE(): number
+    readU56(): number
+    readU56BE(): number
+    readU64(): number
+    readU64BE(): number
+    readBytes(size: number, copy?: boolean): Buffer
+  }
+
+  class Struct {
+    constructor()
+    encode(extra?: object): Buffer
+    decode<T extends Struct>(data: Buffer, extra?: object): T
+    getSize(extra?: object): number
+    fromHex(s: string, extra?: object): this
+    toHex(): string
+    write(bw: BufferWriter, extra?: object): BufferWriter
+    read(br: BufferReader, extra?: object): this
+    static read<T extends Struct>(br: BufferReader, extra?: object): T
+    static decode<T extends Struct>(data: Buffer, extra?: object): T
+    static fromHex<T extends Struct>(s: string, extra?: object): T
+  }
+}

packages/core-utils/test/batch-encoder.spec.ts

@@ -7,23 +7,31 @@ import {
   encodeAppendSequencerBatch,
   decodeAppendSequencerBatch,
   sequencerBatch,
+  BatchType,
+  SequencerBatch,
 } from '../src'
 
-describe('BatchEncoder', () => {
+describe('BatchEncoder', function () {
+  this.timeout(10_000)
+
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const data = require('./fixtures/calldata.json')
+
   describe('appendSequencerBatch', () => {
-    it('should work with the simple case', () => {
+    it('legacy: should work with the simple case', () => {
       const batch = {
         shouldStartAtElement: 0,
         totalElementsToAppend: 0,
         contexts: [],
         transactions: [],
+        type: BatchType.LEGACY,
       }
       const encoded = encodeAppendSequencerBatch(batch)
       const decoded = decodeAppendSequencerBatch(encoded)
       expect(decoded).to.deep.equal(batch)
     })
 
-    it('should work with more complex case', () => {
+    it('legacy: should work with more complex case', () => {
       const batch = {
         shouldStartAtElement: 10,
         totalElementsToAppend: 1,
@@ -36,19 +44,57 @@ describe('BatchEncoder', () => {
           },
         ],
         transactions: ['0x45423400000011', '0x45423400000012'],
+        type: BatchType.LEGACY,
       }
       const encoded = encodeAppendSequencerBatch(batch)
       const decoded = decodeAppendSequencerBatch(encoded)
       expect(decoded).to.deep.equal(batch)
     })
 
-    it('should work with mainnet calldata', () => {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      const data = require('./fixtures/appendSequencerBatch.json')
-      for (const calldata of data.calldata) {
-        const decoded = sequencerBatch.decode(calldata)
-        const encoded = sequencerBatch.encode(decoded)
-        expect(encoded).to.equal(calldata)
+    describe('mainnet data', () => {
+      for (const [hash, calldata] of Object.entries(data)) {
+        // Deserialize the raw calldata
+        const decoded = SequencerBatch.fromHex<SequencerBatch>(
+          calldata as string
+        )
+
+        it(`${hash}`, () => {
+          const encoded = decoded.toHex()
+          expect(encoded).to.deep.equal(calldata)
+
+          const batch = SequencerBatch.decode(decoded.encode())
+          expect(decoded).to.deep.eq(batch)
+        })
+
+        it(`${hash} (compressed)`, () => {
+          // Set the batch type to be zlib so that the batch
+          // is compressed
+          decoded.type = BatchType.ZLIB
+          // Encode a compressed batch
+          const encodedCompressed = decoded.encode()
+          // Decode a compressed batch
+          const decodedPostCompressed =
+            SequencerBatch.decode<SequencerBatch>(encodedCompressed)
+          // Expect that the batch type is detected
+          expect(decodedPostCompressed.type).to.eq(BatchType.ZLIB)
+          // Expect that the contexts match
+          expect(decoded.contexts).to.deep.equal(decodedPostCompressed.contexts)
+          for (const [i, tx] of decoded.transactions.entries()) {
+            const got = decodedPostCompressed.transactions[i]
+            expect(got).to.deep.eq(tx)
+          }
+          // Reserialize the batch as legacy
+          decodedPostCompressed.type = BatchType.LEGACY
+          // Ensure that the original data can be recovered
+          const encoded = decodedPostCompressed.toHex()
+          expect(encoded).to.deep.equal(calldata)
+        })
+
+        it(`${hash}: serialize txs`, () => {
+          for (const tx of decoded.transactions) {
+            tx.toTransaction()
+          }
+        })
       }
     })
 

packages/core-utils/tsconfig.json

 {
-  "extends": "../../tsconfig.json"
+  "extends": "../../tsconfig.json",
+  "typeRoots": ["node_modules/@types", "src/@types"]
 }