op-challenger: Handle Reader reading data and returning EOF in the same call (#9168)

* op-challenger: Implement verification for preimages

Identifies the leaf to challenge, but doesn't yet produce the merkle proofs required and can't actually send the transaction.

* op-challenger: Review feedback.

* op-challenger: Avoid nil

* op-challenger: Handle Reader reading data and returning EOF in the same call.

This is an unusual but valid corner case of io.Reader. Special handling is required to ensure that the state commitment is captured both before and after the last, padding only block.

* fixes

---------
Co-authored-by: refcell.eth <abigger87@gmail.com>

op-challenger/game/keccak/matrix/immediateeof_test.go

+package matrix
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-service/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+type sameCallEOFReader struct {
+	idx  int
+	data []byte
+}
+
+// newSameCallEOFReader returns an io.Reader that returns io.EOF in the same call that returns the final byte of data.
+// This is valid as per io.Reader:
+// An instance of this general case is that a Reader returning
+// a non-zero number of bytes at the end of the input stream may
+// return either err == EOF or err == nil. The next Read should
+// return 0, EOF.
+func newSameCallEOFReader(data []byte) *sameCallEOFReader {
+	return &sameCallEOFReader{data: data}
+}
+
+func (i *sameCallEOFReader) Read(out []byte) (int, error) {
+	end := min(len(i.data), i.idx+len(out))
+	n := copy(out, i.data[i.idx:end])
+	i.idx += n
+	if i.idx >= len(i.data) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+func TestImmediateEofReader(t *testing.T) {
+	rng := rand.New(rand.NewSource(223))
+	data := testutils.RandomData(rng, 100)
+
+	batchSizes := []int{1, 2, 3, 5, 10, 33, 99, 100, 101}
+	for _, size := range batchSizes {
+		size := size
+		t.Run(fmt.Sprintf("Size-%v", size), func(t *testing.T) {
+
+			reader := &sameCallEOFReader{data: data}
+			out := make([]byte, size)
+			actual := make([]byte, 0, len(data))
+			for {
+				n, err := reader.Read(out)
+				actual = append(actual, out[:n]...)
+				if errors.Is(err, io.EOF) {
+					break
+				} else {
+					require.NoError(t, err)
+				}
+			}
+			require.Equal(t, data, actual)
+			n, err := reader.Read(out)
+			require.Zero(t, n)
+			require.ErrorIs(t, err, io.EOF)
+		})
+	}
+}

op-challenger/game/keccak/matrix/matrix.go

@@ -134,6 +134,7 @@ func (d *StateMatrix) absorbNextLeafInput(in io.Reader) ([]byte, error) {
 	for read < types.BlockSize {
 		n, err := in.Read(data[read:])
 		if errors.Is(err, io.EOF) {
+			read += n
 			final = true
 			break
 		} else if err != nil {
@@ -142,6 +143,11 @@ func (d *StateMatrix) absorbNextLeafInput(in io.Reader) ([]byte, error) {
 		read += n
 	}
 	input := data[:read]
+	// Don't add the padding if we read a full block of input data, even if we reached EOF.
+	// Just absorb the full block and return so the caller can capture the state commitment after the block
+	// The next call will read no data from the Reader (already at EOF) and so add the final padding as an
+	// additional block. We can then return EOF to indicate there are no further blocks.
+	final = final && len(input) < types.BlockSize
 	d.absorbLeafInput(input, final)
 	if final {
 		return input, io.EOF

op-challenger/game/keccak/matrix/matrix_test.go

@@ -112,6 +112,34 @@ func TestAbsorbUpTo_ReferenceCommitments(t *testing.T) {
 	}
 }
 
+func TestAbsorbUpTo_ReferenceCommitments_SameCallEOF(t *testing.T) {
+	var tests []testData
+	require.NoError(t, json.Unmarshal(refTests, &tests))
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Ref-%v", i), func(t *testing.T) {
+			s := NewStateMatrix()
+			commitments := []common.Hash{s.StateCommitment()}
+			in := newSameCallEOFReader(test.Input)
+			for {
+				input, err := s.AbsorbUpTo(in, types.BlockSize*3)
+				if errors.Is(err, io.EOF) {
+					commitments = append(commitments, input.Commitments...)
+					break
+				}
+				// Shouldn't get any error except EOF
+				require.NoError(t, err)
+				commitments = append(commitments, input.Commitments...)
+			}
+			actual := s.Hash()
+			expected := crypto.Keccak256Hash(test.Input)
+			require.Equal(t, expected, actual)
+			require.Equal(t, test.Commitments, commitments)
+		})
+	}
+}
+
 func TestAbsorbUpTo_LimitsDataRead(t *testing.T) {
 	s := NewStateMatrix()
 	data := testutils.RandomData(rand.New(rand.NewSource(2424)), types.BlockSize*6+20)
@@ -195,6 +223,36 @@ func TestMatrix_AbsorbNextLeaf(t *testing.T) {
 	}
 }
 
+func TestVerifyPreimage_ReferenceCommitments(t *testing.T) {
+	var tests []testData
+	require.NoError(t, json.Unmarshal(refTests, &tests))
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Ref-%v", i), func(t *testing.T) {
+			// Exclude the empty state commitment
+			challenge, err := Challenge(bytes.NewReader(test.Input), test.Commitments[1:])
+			require.ErrorIs(t, err, ErrValid)
+			require.Equal(t, types.Challenge{}, challenge)
+		})
+	}
+}
+
+func TestVerifyPreimage_ReferenceCommitments_SameCallEOF(t *testing.T) {
+	var tests []testData
+	require.NoError(t, json.Unmarshal(refTests, &tests))
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Ref-%v", i), func(t *testing.T) {
+			// Exclude the empty state commitment
+			challenge, err := Challenge(newSameCallEOFReader(test.Input), test.Commitments[1:])
+			require.ErrorIs(t, err, ErrValid)
+			require.Equal(t, types.Challenge{}, challenge)
+		})
+	}
+}
+
 func TestVerifyPreimage(t *testing.T) {
 	preimage := testutils.RandomData(rand.New(rand.NewSource(2323)), 1024)
 	validCommitments := func() []common.Hash {

op-challenger/game/keccak/matrix/testdata/commitments.json

@@ -861,5 +861,27 @@
       "0xf6785ff09114b843cccff1e119100ca5ec235725ef2763573e1f3063bf48b886",
       "0x20642552f0e7ef28ff28252b3839f0fa323f46792bd103baa9778684f737cda3"
     ]
+  },
+  {
+    "input": "O8NUg0KaDo18ybTKajXM/sgqEYS37+lewPhGV/2sMAX6bu6pD+hu97cVqUkoazqhtXPFvnwPFy/jQjR+LFOXFZvvjCRgFRQvzzKOb8DJ0fqCRLgr9kiHN9LpqHXVhyHhhlRkem0+AIzvEmyqbgj5pbDQsr0gtBBZYN/IwUYeggMVEhM/rDKEiLf1Z5E/xs1a5ukkKJwXBVAioEmG81qwOBn9FqHWIrclhCUpqZRfRIZPp8ZXl64RTSIXhGibMujQipVQuXUsAOd4QE4HjirdI9SfzsLMesYimfC/eedEkhjL0z9+sjhgp5wJY84LBiNI6ZwrzNGke/zdySctbfKbgj2q6ECX3jy6lbeiNThEdLfO1t+IHM7rijOlNw/r8Up3YgH5ExUuuAOUJ5B8OuPXyMfnl484FISrdt0XGpNJpS66b1XHmf7p3TmFLlnO7/23uN57KiNdpe0RaB8R+Y5IzmcWrzR67wcbjbBTbjbAD6w1QIWVx6iUVSfciQIgw7WdYK1C6PXxUMEflpAnV0POZSMu++sbYrxJhlGArYgVbSH4i93eENLOJytOfnKXYzirqsHyCjHa55FqYBqC4ogkXXHBIzE90x9n2bNpc2IHYXIUfDcrkZSQFBnyze2uRsYx8dBXi+Pd2pLSFc3l0WnUCRXc4LCMFKEKWy/6fPHAUTy569sNCQd1E19Fx96UAQOsLRAtiHKp9XwCQYFhnPY2j61MYnsnV1Eh+cBc4J8ofY4mK4r7OBY9ze9+qrP95CCxyDJUIHDzDySR05MlXTJwwmA1a2X4r45shLLgZEF6EWsLwSvibme0tjvzczsfIwNGAmvg2gx4Uajr6kBelBRAobURalVBBx0SLghEjf1pXftmDkmRJipoV7tUVNK4tYEm5VVAZ95AAw==",
+    "commitments": [
+      "0xee0a1a26c607ab52c6308165995365f7951a185fccca4b76c847b8860d9fea7a",
+      "0xbedfbe751c8d3913a76bb030bdcad312ce8c70aeabfaefcf491dafe740946ca5",
+      "0x74393cbd5f8764be54385a8c7f9442358ae2580f33aebc2597ffa42949172a34",
+      "0xc881ce64585de627f538376cae9513ca5f4efaa2ab255ce3fa5b15e890223982",
+      "0xa6ad654807a4f8de3b78d7fa389dc3bd31bbbb5bef24e8e8174f75c6f9ac5c6d",
+      "0x71ca4f0903770f2035f3650a40ac8b23d00aaffd7b155a8be5bd02ff1c944181"
+    ]
+  },
+  {
+    "input": "O8NUg0KaDo18ybTKajXM/sgqEYS37+lewPhGV/2sMAX6bu6pD+hu97cVqUkoazqhtXPFvnwPFy/jQjR+LFOXFZvvjCRgFRQvzzKOb8DJ0fqCRLgr9kiHN9LpqHXVhyHhhlRkem0+AIzvEmyqbgj5pbDQsr0gtBBZYN/IwUYeggMVEhM/rDKEiLf1Z5E/xs1a5ukkKJwXBVAioEmG81qwOBn9FqHWIrclhCUpqZRfRIZPp8ZXl64RTSIXhGibMujQipVQuXUsAOd4QE4HjirdI9SfzsLMesYimfC/eedEkhjL0z9+sjhgp5wJY84LBiNI6ZwrzNGke/zdySctbfKbgj2q6ECX3jy6lbeiNThEdLfO1t+IHM7rijOlNw/r8Up3YgH5ExUuuAOUJ5B8OuPXyMfnl484FISrdt0XGpNJpS66b1XHmf7p3TmFLlnO7/23uN57KiNdpe0RaB8R+Y5IzmcWrzR67wcbjbBTbjbAD6w1QIWVx6iUVSfciQIgw7WdYK1C6PXxUMEflpAnV0POZSMu++sbYrxJhlGArYgVbSH4i93eENLOJytOfnKXYzirqsHyCjHa55FqYBqC4ogkXXHBIzE90x9n2bNpc2IHYXIUfDcrkZSQFBnyze2uRsYx8dBXi+Pd2pLSFc3l0WnUCRXc4LCMFKEKWy/6fPHAUTy569sNCQd1E19Fx96UAQOsLRAtiHKp9XwCQYFhnPY2j61MYnsnV1Eh+cBc4J8ofY4mK4r7OBY9ze9+qrP95CCxyDJUIHDzDySR05MlXTJwwmA1a2X4r45shLLgZEF6EWsLwSvibme0tjvzczsfIwNGAmvg2gx4Uajr6kBelBRAobURalVBBx0SLghEjf1pXftmDkmRJipoV7tUVNK4tYEm5VVAZ95A",
+    "commitments": [
+      "0xee0a1a26c607ab52c6308165995365f7951a185fccca4b76c847b8860d9fea7a",
+      "0xbedfbe751c8d3913a76bb030bdcad312ce8c70aeabfaefcf491dafe740946ca5",
+      "0x74393cbd5f8764be54385a8c7f9442358ae2580f33aebc2597ffa42949172a34",
+      "0xc881ce64585de627f538376cae9513ca5f4efaa2ab255ce3fa5b15e890223982",
+      "0xa6ad654807a4f8de3b78d7fa389dc3bd31bbbb5bef24e8e8174f75c6f9ac5c6d",
+      "0x8c3c5310bc148f9c71d9c0a8771479fb87c6fd9d9595ba60ded7b8eba1d6a483"
+    ]
   }
 ]