op-challenger: Add keccak sponge implementation (#9015)

* op-challenger: Add keccak sponge impl

Provides a StateMatrix implementation that can calculate the state commitment after each permutation.

* op-challenger: Add randomly generated reference tests.

* op-challenger: Split library code to a separate file.

* op-challenger: Expose PackState method

* op-challenger: Add reader based method to make it easier to split up data correctly.

* op-challenger: Fix fuzz makefile target.

* op-challenger: Include full license to comply with source distribution requirements.

.circleci/config.yml

@@ -1468,6 +1468,11 @@ workflows:
       - go-lint: # we combine most of the go-lint work for two reasons: (1) warm up the Go build cache, (2) reduce sum of lint time
           name: op-stack-go-lint
           requires: [ "go-mod-tidy" ]
+      - fuzz-golang:
+          name: op-challenger-fuzz
+          package_name: op-challenger
+          on_changes: op-challenger
+          requires: ["op-stack-go-lint"]
       - fuzz-golang:
           name: op-node-fuzz
           package_name: op-node

op-challenger/Makefile

@@ -10,6 +10,9 @@ LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 op-challenger:
 	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/op-challenger ./cmd
 
+fuzz:
+	go test -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzKeccak ./game/keccak/matrix
+
 clean:
 	rm bin/op-challenger
 

op-challenger/game/keccak/matrix/matrix.go

+package matrix
+
+import (
+	"errors"
+	"io"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// StateMatrix implements a stateful keccak sponge with the ability to create state commitments after each permutation
+type StateMatrix struct {
+	s *state
+}
+
+// LeafSize is the size in bytes required for leaf data.
+const LeafSize = 136
+
+var uint256Size = 32
+
+// NewStateMatrix creates a new state matrix initialized with the initial, zero keccak block.
+func NewStateMatrix() *StateMatrix {
+	return &StateMatrix{s: newLegacyKeccak256()}
+}
+
+// StateCommitment returns the state commitment for the current state matrix.
+// Additional data may be absorbed after calling this method.
+func (d *StateMatrix) StateCommitment() common.Hash {
+	buf := d.PackState()
+	return crypto.Keccak256Hash(buf)
+}
+
+// PackState packs the state in to the solidity ABI encoding required for the state matrix
+func (d *StateMatrix) PackState() []byte {
+	buf := make([]byte, 0, len(d.s.a)*uint256Size)
+	for _, v := range d.s.a {
+		buf = append(buf, math.U256Bytes(new(big.Int).SetUint64(v))...)
+	}
+	return buf
+}
+
+// AbsorbNextLeaf reads up to [LeafSize] bytes from in and absorbs them into the state matrix.
+// If EOF is reached while reading, the state matrix is finalized and [io.EOF] is returned.
+func (d *StateMatrix) AbsorbNextLeaf(in io.Reader) error {
+	data := make([]byte, LeafSize)
+	read := 0
+	final := false
+	for read < LeafSize {
+		n, err := in.Read(data[read:])
+		if errors.Is(err, io.EOF) {
+			final = true
+			break
+		} else if err != nil {
+			return err
+		}
+		read += n
+	}
+	d.AbsorbLeaf(data[:read], final)
+	if final {
+		return io.EOF
+	}
+	return nil
+}
+
+// AbsorbLeaf absorbs the specified data into the keccak sponge.
+// If final is true, the data is padded to the required length, otherwise it must be exactly
+// LeafSize bytes.
+func (d *StateMatrix) AbsorbLeaf(data []byte, final bool) {
+	if !final && len(data) != LeafSize {
+		panic("sha3: Incorrect leaf data length")
+	}
+	_, _ = d.s.Write(data[:])
+	if final {
+		d.s.padAndPermute(d.s.dsbyte)
+	}
+}
+
+// Hash finalizes the keccak permutation and returns the final hash.
+// No further leaves can be absorbed after this is called
+func (d *StateMatrix) Hash() (h common.Hash) {
+	_, _ = d.s.Read(h[:])
+	return h
+}

op-challenger/game/keccak/matrix/matrix_test.go

+package matrix
+
+import (
+	"bytes"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed testdata/commitments.json
+var refTests []byte
+
+func TestStateCommitment(t *testing.T) {
+	tests := []struct {
+		expectedPacked string
+		matrix         []uint64 // Automatically padded with 0s to the required length
+	}{
+		{
+			expectedPacked: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+		},
+		{
+			expectedPacked: "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f0000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000013000000000000000000000000000000000000000000000000000000000000001400000000000000000000000000000000000000000000000000000000000000150000000000000000000000000000000000000000000000000000000000000016000000000000000000000000000000000000000000000000000000000000001700000000000000000000000000000000000000000000000000000000000000180000000000000000000000000000000000000000000000000000000000000019",
+			matrix:         []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25},
+		},
+		{
+			expectedPacked: "000000000000000000000000000000000000000000000000ffffffffffffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+			matrix:         []uint64{18446744073709551615},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run("", func(t *testing.T) {
+			state := NewStateMatrix()
+			copy(state.s.a[:], test.matrix)
+			expected := crypto.Keccak256Hash(common.Hex2Bytes(test.expectedPacked))
+			actual := state.StateCommitment()
+			require.Equal(t, test.expectedPacked, common.Bytes2Hex(state.PackState()))
+			require.Equal(t, expected, actual)
+		})
+	}
+}
+
+type testData struct {
+	Input       []byte        `json:"input"`
+	Commitments []common.Hash `json:"commitments"`
+}
+
+func TestReferenceCommitments(t *testing.T) {
+	var tests []testData
+	require.NoError(t, json.Unmarshal(refTests, &tests))
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Ref-%v", i), func(t *testing.T) {
+			s := NewStateMatrix()
+			commitments := []common.Hash{s.StateCommitment()}
+			for i := 0; i < len(test.Input); i += LeafSize {
+				end := min(i+LeafSize, len(test.Input))
+				s.AbsorbLeaf(test.Input[i:end], end == len(test.Input))
+				commitments = append(commitments, s.StateCommitment())
+			}
+			if len(test.Input) == 0 {
+				s.AbsorbLeaf(nil, true)
+				commitments = append(commitments, s.StateCommitment())
+			}
+			actual := s.Hash()
+			expected := crypto.Keccak256Hash(test.Input)
+			require.Equal(t, expected, actual)
+			require.Equal(t, test.Commitments, commitments)
+		})
+	}
+}
+
+func TestReferenceCommitmentsFromReader(t *testing.T) {
+	var tests []testData
+	require.NoError(t, json.Unmarshal(refTests, &tests))
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Ref-%v", i), func(t *testing.T) {
+			s := NewStateMatrix()
+			commitments := []common.Hash{s.StateCommitment()}
+			in := bytes.NewReader(test.Input)
+			for {
+				err := s.AbsorbNextLeaf(in)
+				if errors.Is(err, io.EOF) {
+					commitments = append(commitments, s.StateCommitment())
+					break
+				}
+				// Shouldn't get any error except EOF
+				require.NoError(t, err)
+				commitments = append(commitments, s.StateCommitment())
+			}
+			actual := s.Hash()
+			expected := crypto.Keccak256Hash(test.Input)
+			require.Equal(t, expected, actual)
+			require.Equal(t, test.Commitments, commitments)
+		})
+	}
+}
+
+func FuzzKeccak(f *testing.F) {
+	f.Fuzz(func(t *testing.T, number, time uint64, data []byte) {
+		s := NewStateMatrix()
+		for i := 0; i < len(data); i += LeafSize {
+			end := min(i+LeafSize, len(data))
+			s.AbsorbLeaf(data[i:end], end == len(data))
+		}
+		actual := s.Hash()
+		expected := crypto.Keccak256Hash(data)
+		require.Equal(t, expected, actual)
+	})
+}