Skip to content

N
nebula
Commit 2b34a1ee authored by inphi's avatar inphi
Browse files
Options

removed eth_call sync code; cache mod latest blocks

parent 1cf8b15d
Hide whitespace changes
Inline Side-by-side
Showing with 377 additions and 376 deletions
go/proxyd/backend.go
View file @ 2b34a1ee
......@@ -16,9 +16,7 @@ import (
"strings"
"time"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/gorilla/websocket"
"github.com/prometheus/client_golang/prometheus"
)
......@@ -86,7 +84,6 @@ type Backend struct {
authPassword string
rateLimiter RateLimiter
client *http.Client
blockNumberLVC *EthLastValueCache
dialer *websocket.Dialer
maxRetries int
maxResponseSize int64
......@@ -169,7 +166,7 @@ func NewBackend(
wsURL string,
rateLimiter RateLimiter,
opts ...BackendOpt,
) (*Backend, error) {
) *Backend {
backend := &Backend{
Name: name,
rpcURL: rpcURL,
......@@ -186,28 +183,11 @@ func NewBackend(
opt(backend)
}
rpcClient, err := rpc.DialHTTPWithClient(rpcURL, backend.client)
if err != nil {
return nil, err
}
backend.blockNumberLVC = newLVC(ethclient.NewClient(rpcClient), func(ctx context.Context, client *ethclient.Client) (interface{}, error) {
blockNumber, err := client.BlockNumber(ctx)
return blockNumber, err
})
if !backend.stripTrailingXFF && backend.proxydIP == "" {
log.Warn("proxied requests' XFF header will not contain the proxyd ip address")
}
return backend, nil
}
func (b *Backend) Start() {
b.blockNumberLVC.Start()
}
func (b *Backend) Stop() {
b.blockNumberLVC.Stop()
return backend
}
func (b *Backend) Forward(ctx context.Context, req *RPCReq) (*RPCRes, error) {
......@@ -288,14 +268,6 @@ func (b *Backend) ProxyWS(clientConn *websocket.Conn, methodWhitelist *StringSet
return NewWSProxier(b, clientConn, backendConn, methodWhitelist), nil
}
func (b *Backend) BlockNumber() uint64 {
var blockNum uint64
if val := b.blockNumberLVC.Read(); val != nil {
blockNum = val.(uint64)
}
return blockNum
}
func (b *Backend) Online() bool {
online, err := b.rateLimiter.IsBackendOnline(b.Name)
if err != nil {
......@@ -423,16 +395,13 @@ type BackendGroup struct {
Backends []*Backend
}
func (b *BackendGroup) Forward(ctx context.Context, rpcReq *RPCReq) (*RPCRes, uint64, error) {
func (b *BackendGroup) Forward(ctx context.Context, rpcReq *RPCReq) (*RPCRes, error) {
rpcRequestsTotal.Inc()
for _, back := range b.Backends {
// The blockNum must precede the forwarded RPC to establish a synchronization point
blockNum := back.BlockNumber()
res, err := back.Forward(ctx, rpcReq)
if errors.Is(err, ErrMethodNotWhitelisted) {
return nil, 0, err
return nil, err
}
if errors.Is(err, ErrBackendOffline) {
log.Warn(
......@@ -462,11 +431,11 @@ func (b *BackendGroup) Forward(ctx context.Context, rpcReq *RPCReq) (*RPCRes, ui
)
continue
}
return res, blockNum, nil
return res, nil
}
RecordUnserviceableRequest(ctx, RPCRequestSourceHTTP)
return nil, 0, ErrNoBackends
return nil, ErrNoBackends
}
func (b *BackendGroup) ProxyWS(ctx context.Context, clientConn *websocket.Conn, methodWhitelist *StringSet) (*WSProxier, error) {
......
go/proxyd/cache.go
View file @ 2b34a1ee
......@@ -4,13 +4,13 @@ import (
"context"
"github.com/go-redis/redis/v8"
"github.com/golang/snappy"
lru "github.com/hashicorp/golang-lru"
)
type Cache interface {
Get(ctx context.Context, key string) (string, error)
Put(ctx context.Context, key string, value string) error
Remove(ctx context.Context, key string) error
}
const (
......@@ -39,11 +39,6 @@ func (c *cache) Put(ctx context.Context, key string, value string) error {
return nil
}
func (c *cache) Remove(ctx context.Context, key string) error {
c.lru.Remove(key)
return nil
}
type redisCache struct {
rdb *redis.Client
}
......@@ -79,12 +74,32 @@ func (c *redisCache) Put(ctx context.Context, key string, value string) error {
return err
}
func (c *redisCache) Remove(ctx context.Context, key string) error {
err := c.rdb.Del(ctx, key).Err()
type cacheWithCompression struct {
cache Cache
}
func newCacheWithCompression(cache Cache) *cacheWithCompression {
return &cacheWithCompression{cache}
}
func (c *cacheWithCompression) Get(ctx context.Context, key string) (string, error) {
encodedVal, err := c.cache.Get(ctx, key)
if err != nil {
RecordRedisError("CacheDel")
return "", err
}
return err
if encodedVal == "" {
return "", nil
}
val, err := snappy.Decode(nil, []byte(encodedVal))
if err != nil {
return "", err
}
return string(val), nil
}
func (c *cacheWithCompression) Put(ctx context.Context, key string, value string) error {
encodedVal := snappy.Encode(nil, []byte(value))
return c.cache.Put(ctx, key, string(encodedVal))
}
type GetLatestBlockNumFn func(ctx context.Context) (uint64, error)
......@@ -92,11 +107,7 @@ type GetLatestGasPriceFn func(ctx context.Context) (uint64, error)
type RPCCache interface {
GetRPC(ctx context.Context, req *RPCReq) (*RPCRes, error)
// The blockNumberSync is used to enforce Sequential Consistency for cache invalidation. We make the following assumptions to do this:
// 1. blockNumberSync is monotonically increasing (sans reorgs)
// 2. blockNumberSync is ordered before block state of the RPCRes
PutRPC(ctx context.Context, req *RPCReq, res *RPCRes, blockNumberSync uint64) error
PutRPC(ctx context.Context, req *RPCReq, res *RPCRes) error
}
type rpcCache struct {
......@@ -104,15 +115,15 @@ type rpcCache struct {
handlers map[string]RPCMethodHandler
}
func newRPCCache(cache Cache, getLatestBlockNumFn GetLatestBlockNumFn, getLatestGasPriceFn GetLatestGasPriceFn) RPCCache {
func newRPCCache(cache Cache, getLatestBlockNumFn GetLatestBlockNumFn, getLatestGasPriceFn GetLatestGasPriceFn, numBlockConfirmations int) RPCCache {
handlers := map[string]RPCMethodHandler{
"eth_chainId": &StaticMethodHandler{},
"net_version": &StaticMethodHandler{},
"eth_getBlockByNumber": &EthGetBlockByNumberMethodHandler{cache, getLatestBlockNumFn},
"eth_getBlockRange": &EthGetBlockRangeMethodHandler{cache, getLatestBlockNumFn},
"eth_getBlockByNumber": &EthGetBlockByNumberMethodHandler{cache, getLatestBlockNumFn, numBlockConfirmations},
"eth_getBlockRange": &EthGetBlockRangeMethodHandler{cache, getLatestBlockNumFn, numBlockConfirmations},
"eth_blockNumber": &EthBlockNumberMethodHandler{getLatestBlockNumFn},
"eth_gasPrice": &EthGasPriceMethodHandler{getLatestGasPriceFn},
"eth_call": &EthCallMethodHandler{cache, getLatestBlockNumFn},
"eth_call": &EthCallMethodHandler{cache, getLatestBlockNumFn, numBlockConfirmations},
}
return &rpcCache{
cache: cache,
......@@ -136,10 +147,10 @@ func (c *rpcCache) GetRPC(ctx context.Context, req *RPCReq) (*RPCRes, error) {
return res, err
}
func (c *rpcCache) PutRPC(ctx context.Context, req *RPCReq, res *RPCRes, blockNumberSync uint64) error {
func (c *rpcCache) PutRPC(ctx context.Context, req *RPCReq, res *RPCRes) error {
handler := c.handlers[req.Method]
if handler == nil {
return nil
}
return handler.PutRPCMethod(ctx, req, res, blockNumberSync)
return handler.PutRPCMethod(ctx, req, res)
}
go/proxyd/cache_test.go
View file @ 2b34a1ee
......@@ -5,11 +5,12 @@ import (
"math"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/require"
)
const numBlockConfirmations = 10
func TestRPCCacheImmutableRPCs(t *testing.T) {
const blockHead = math.MaxUint64
ctx := context.Background()
......@@ -17,7 +18,7 @@ func TestRPCCacheImmutableRPCs(t *testing.T) {
getBlockNum := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), getBlockNum, nil)
cache := newRPCCache(newMemoryCache(), getBlockNum, nil, numBlockConfirmations)
ID := []byte(strconv.Itoa(1))
rpcs := []struct {
......@@ -111,7 +112,7 @@ func TestRPCCacheImmutableRPCs(t *testing.T) {
for _, rpc := range rpcs {
t.Run(rpc.name, func(t *testing.T) {
err := cache.PutRPC(ctx, rpc.req, rpc.res, blockHead)
err := cache.PutRPC(ctx, rpc.req, rpc.res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, rpc.req)
......@@ -133,7 +134,7 @@ func TestRPCCacheBlockNumber(t *testing.T) {
getBlockNum := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), getBlockNum, getGasPrice)
cache := newRPCCache(newMemoryCache(), getBlockNum, getGasPrice, numBlockConfirmations)
req := &RPCReq{
JSONRPC: "2.0",
......@@ -146,12 +147,17 @@ func TestRPCCacheBlockNumber(t *testing.T) {
ID: ID,
}
err := cache.PutRPC(ctx, req, res, blockHead)
err := cache.PutRPC(ctx, req, res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
blockHead = 0x1001
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, &RPCRes{JSONRPC: "2.0", Result: `0x1001`, ID: ID}, cachedRes)
}
func TestRPCCacheGasPrice(t *testing.T) {
......@@ -166,7 +172,7 @@ func TestRPCCacheGasPrice(t *testing.T) {
getBlockNum := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), getBlockNum, getGasPrice)
cache := newRPCCache(newMemoryCache(), getBlockNum, getGasPrice, numBlockConfirmations)
req := &RPCReq{
JSONRPC: "2.0",
......@@ -179,12 +185,17 @@ func TestRPCCacheGasPrice(t *testing.T) {
ID: ID,
}
err := cache.PutRPC(ctx, req, res, blockHead)
err := cache.PutRPC(ctx, req, res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
gasPrice = 0x101
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, &RPCRes{JSONRPC: "2.0", Result: `0x101`, ID: ID}, cachedRes)
}
func TestRPCCacheUnsupportedMethod(t *testing.T) {
......@@ -194,7 +205,7 @@ func TestRPCCacheUnsupportedMethod(t *testing.T) {
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
cache := newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations)
ID := []byte(strconv.Itoa(1))
req := &RPCReq{
......@@ -208,7 +219,7 @@ func TestRPCCacheUnsupportedMethod(t *testing.T) {
ID: ID,
}
err := cache.PutRPC(ctx, req, res, blockHead)
err := cache.PutRPC(ctx, req, res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, req)
......@@ -219,12 +230,11 @@ func TestRPCCacheUnsupportedMethod(t *testing.T) {
func TestRPCCacheEthGetBlockByNumber(t *testing.T) {
ctx := context.Background()
var blockHead uint64 = 100
var blockHead uint64
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
resetCache := func() { cache = newRPCCache(newMemoryCache(), fn, nil) }
makeCache := func() RPCCache { return newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations) }
ID := []byte(strconv.Itoa(1))
req := &RPCReq{
......@@ -250,22 +260,27 @@ func TestRPCCacheEthGetBlockByNumber(t *testing.T) {
ID: ID,
}
require.NoError(t, cache.PutRPC(ctx, req, res, blockHead))
require.NoError(t, cache.PutRPC(ctx, req2, res2, blockHead))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
cachedRes, err = cache.GetRPC(ctx, req2)
require.NoError(t, err)
require.Equal(t, res2, cachedRes)
// scenario: input references block that has not yet finalized
resetCache()
blockHead = 0xc
require.NoError(t, cache.PutRPC(ctx, req, res, blockHead))
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
t.Run("set multiple finalized blocks", func(t *testing.T) {
blockHead = 100
cache := makeCache()
require.NoError(t, cache.PutRPC(ctx, req, res))
require.NoError(t, cache.PutRPC(ctx, req2, res2))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
cachedRes, err = cache.GetRPC(ctx, req2)
require.NoError(t, err)
require.Equal(t, res2, cachedRes)
})
t.Run("unconfirmed block", func(t *testing.T) {
blockHead = 0xc
cache := makeCache()
require.NoError(t, cache.PutRPC(ctx, req, res))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
})
}
func TestRPCCacheEthGetBlockByNumberForRecentBlocks(t *testing.T) {
......@@ -275,7 +290,7 @@ func TestRPCCacheEthGetBlockByNumberForRecentBlocks(t *testing.T) {
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
cache := newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations)
ID := []byte(strconv.Itoa(1))
rpcs := []struct {
......@@ -315,7 +330,7 @@ func TestRPCCacheEthGetBlockByNumberForRecentBlocks(t *testing.T) {
for _, rpc := range rpcs {
t.Run(rpc.name, func(t *testing.T) {
err := cache.PutRPC(ctx, rpc.req, rpc.res, blockHead)
err := cache.PutRPC(ctx, rpc.req, rpc.res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, rpc.req)
......@@ -332,7 +347,7 @@ func TestRPCCacheEthGetBlockByNumberInvalidRequest(t *testing.T) {
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
cache := newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations)
ID := []byte(strconv.Itoa(1))
req := &RPCReq{
......@@ -347,7 +362,7 @@ func TestRPCCacheEthGetBlockByNumberInvalidRequest(t *testing.T) {
ID: ID,
}
err := cache.PutRPC(ctx, req, res, blockHead)
err := cache.PutRPC(ctx, req, res)
require.Error(t, err)
cachedRes, err := cache.GetRPC(ctx, req)
......@@ -358,46 +373,51 @@ func TestRPCCacheEthGetBlockByNumberInvalidRequest(t *testing.T) {
func TestRPCCacheEthGetBlockRange(t *testing.T) {
ctx := context.Background()
var blockHead uint64 = 0x1000
var blockHead uint64
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
makeCache := func() RPCCache { return newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations) }
ID := []byte(strconv.Itoa(1))
req := &RPCReq{
JSONRPC: "2.0",
Method: "eth_getBlockRange",
Params: []byte(`["0x1", "0x10", false]`),
ID: ID,
}
res := &RPCRes{
JSONRPC: "2.0",
Result: `[{"number": "0x1"}, {"number": "0x10"}]`,
ID: ID,
}
require.NoError(t, cache.PutRPC(ctx, req, res, blockHead))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
// scenario: input references block that has not yet finalized
req = &RPCReq{
JSONRPC: "2.0",
Method: "eth_getBlockRange",
Params: []byte(`["0x1", "0x1000", false]`),
ID: ID,
}
res = &RPCRes{
JSONRPC: "2.0",
Result: `[{"number": "0x1"}, {"number": "0x2"}]`,
ID: ID,
}
require.NoError(t, cache.PutRPC(ctx, req, res, blockHead))
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
t.Run("finalized block", func(t *testing.T) {
req := &RPCReq{
JSONRPC: "2.0",
Method: "eth_getBlockRange",
Params: []byte(`["0x1", "0x10", false]`),
ID: ID,
}
res := &RPCRes{
JSONRPC: "2.0",
Result: `[{"number": "0x1"}, {"number": "0x10"}]`,
ID: ID,
}
blockHead = 0x1000
cache := makeCache()
require.NoError(t, cache.PutRPC(ctx, req, res))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
})
t.Run("unconfirmed block", func(t *testing.T) {
cache := makeCache()
req := &RPCReq{
JSONRPC: "2.0",
Method: "eth_getBlockRange",
Params: []byte(`["0x1", "0x1000", false]`),
ID: ID,
}
res := &RPCRes{
JSONRPC: "2.0",
Result: `[{"number": "0x1"}, {"number": "0x2"}]`,
ID: ID,
}
require.NoError(t, cache.PutRPC(ctx, req, res))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
})
}
func TestRPCCacheEthGetBlockRangeForRecentBlocks(t *testing.T) {
......@@ -407,7 +427,7 @@ func TestRPCCacheEthGetBlockRangeForRecentBlocks(t *testing.T) {
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
cache := newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations)
ID := []byte(strconv.Itoa(1))
rpcs := []struct {
......@@ -461,7 +481,7 @@ func TestRPCCacheEthGetBlockRangeForRecentBlocks(t *testing.T) {
for _, rpc := range rpcs {
t.Run(rpc.name, func(t *testing.T) {
err := cache.PutRPC(ctx, rpc.req, rpc.res, blockHead)
err := cache.PutRPC(ctx, rpc.req, rpc.res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, rpc.req)
......@@ -478,7 +498,7 @@ func TestRPCCacheEthGetBlockRangeInvalidRequest(t *testing.T) {
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
cache := newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations)
ID := []byte(strconv.Itoa(1))
rpcs := []struct {
......@@ -518,7 +538,7 @@ func TestRPCCacheEthGetBlockRangeInvalidRequest(t *testing.T) {
for _, rpc := range rpcs {
t.Run(rpc.name, func(t *testing.T) {
err := cache.PutRPC(ctx, rpc.req, rpc.res, blockHead)
err := cache.PutRPC(ctx, rpc.req, rpc.res)
require.Error(t, err)
cachedRes, err := cache.GetRPC(ctx, rpc.req)
......@@ -531,18 +551,18 @@ func TestRPCCacheEthGetBlockRangeInvalidRequest(t *testing.T) {
func TestRPCCacheEthCall(t *testing.T) {
ctx := context.Background()
var blockHead uint64 = 0x1000
var blockHead uint64
fn := func(ctx context.Context) (uint64, error) {
return blockHead, nil
}
cache := newRPCCache(newMemoryCache(), fn, nil)
resetCache := func() { cache = newRPCCache(newMemoryCache(), fn, nil) }
makeCache := func() RPCCache { return newRPCCache(newMemoryCache(), fn, nil, numBlockConfirmations) }
ID := []byte(strconv.Itoa(1))
req := &RPCReq{
JSONRPC: "2.0",
Method: "eth_call",
Params: []byte(`[{"to": "0xDEADBEEF", "data": "0x1"}, "latest"]`),
Params: []byte(`[{"to": "0xDEADBEEF", "data": "0x1"}, "0x10"]`),
ID: ID,
}
res := &RPCRes{
......@@ -551,38 +571,52 @@ func TestRPCCacheEthCall(t *testing.T) {
ID: ID,
}
err := cache.PutRPC(ctx, req, res, blockHead)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
// scenario: no new block, but we've exceeded cacheTTL
resetCache()
cacheTTL = 24 * time.Hour
err = cache.PutRPC(ctx, req, res, blockHead)
require.NoError(t, err)
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
// scenario: new block, but cached TTL is live
resetCache()
cacheTTL = 24 * time.Hour
err = cache.PutRPC(ctx, req, res, blockHead)
require.NoError(t, err)
blockHead += 1 // new block
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
// scenario: new block, cache TTL exceeded; cache invalidation
resetCache()
cacheTTL = 0 * time.Second
err = cache.PutRPC(ctx, req, res, blockHead)
require.NoError(t, err)
blockHead += 1 // new block
cachedRes, err = cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
t.Run("finalized block", func(t *testing.T) {
blockHead = 0x100
cache := makeCache()
err := cache.PutRPC(ctx, req, res)
require.NoError(t, err)
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Equal(t, res, cachedRes)
})
t.Run("unconfirmed block", func(t *testing.T) {
blockHead = 0x10
cache := makeCache()
require.NoError(t, cache.PutRPC(ctx, req, res))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
})
t.Run("latest block", func(t *testing.T) {
blockHead = 0x100
req := &RPCReq{
JSONRPC: "2.0",
Method: "eth_call",
Params: []byte(`[{"to": "0xDEADBEEF", "data": "0x1"}, "latest"]`),
ID: ID,
}
cache := makeCache()
require.NoError(t, cache.PutRPC(ctx, req, res))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
})
t.Run("pending block", func(t *testing.T) {
blockHead = 0x100
req := &RPCReq{
JSONRPC: "2.0",
Method: "eth_call",
Params: []byte(`[{"to": "0xDEADBEEF", "data": "0x1"}, "pending"]`),
ID: ID,
}
cache := makeCache()
require.NoError(t, cache.PutRPC(ctx, req, res))
cachedRes, err := cache.GetRPC(ctx, req)
require.NoError(t, err)
require.Nil(t, cachedRes)
})
}
go/proxyd/config.go
View file @ 2b34a1ee
......@@ -15,8 +15,9 @@ type ServerConfig struct {
}
type CacheConfig struct {
Enabled bool `toml:"enabled"`
BlockSyncRPCURL string `toml:"block_sync_rpc_url"`
Enabled bool `toml:"enabled"`
BlockSyncRPCURL string `toml:"block_sync_rpc_url"`
NumBlockConfirmations int `toml:"num_block_confirmations"`
}
type RedisConfig struct {
......
go/proxyd/lvc.go
View file @ 2b34a1ee
......@@ -2,7 +2,6 @@ package proxyd
import (
"context"
"sync"
"time"
"github.com/ethereum/go-ethereum/ethclient"
......@@ -11,20 +10,21 @@ import (
const cacheSyncRate = 1 * time.Second
type lvcUpdateFn func(context.Context, *ethclient.Client) (interface{}, error)
type lvcUpdateFn func(context.Context, *ethclient.Client) (string, error)
type EthLastValueCache struct {
client *ethclient.Client
cache Cache
key string
updater lvcUpdateFn
quit chan struct{}
mutex sync.RWMutex
value interface{}
}
func newLVC(client *ethclient.Client, updater lvcUpdateFn) *EthLastValueCache {
func newLVC(client *ethclient.Client, cache Cache, cacheKey string, updater lvcUpdateFn) *EthLastValueCache {
return &EthLastValueCache{
client: client,
cache: cache,
key: cacheKey,
updater: updater,
quit: make(chan struct{}),
}
......@@ -38,15 +38,18 @@ func (h *EthLastValueCache) Start() {
for {
select {
case <-ticker.C:
lvcPollTimeGauge.WithLabelValues(h.key).SetToCurrentTime()
value, err := h.getUpdate()
if err != nil {
log.Error("error retrieving latest value", "error", err)
log.Error("error retrieving latest value", "key", h.key, "error", err)
continue
}
log.Trace("polling latest value", "value", value)
h.mutex.Lock()
h.value = value
h.mutex.Unlock()
if err := h.cache.Put(context.Background(), h.key, value); err != nil {
log.Error("error writing last value to cache", "key", h.key, "error", err)
}
case <-h.quit:
return
......@@ -55,31 +58,30 @@ func (h *EthLastValueCache) Start() {
}()
}
func (h *EthLastValueCache) getUpdate() (interface{}, error) {
func (h *EthLastValueCache) getUpdate() (string, error) {
const maxRetries = 5
var err error
for i := 0; i <= maxRetries; i++ {
var value interface{}
var value string
value, err = h.updater(context.Background(), h.client)
if err != nil {
backoff := calcBackoff(i)
log.Warn("http operation failed. retrying...", "error", err, "backoff", backoff)
lvcErrorsTotal.WithLabelValues(h.key).Inc()
time.Sleep(backoff)
continue
}
return value, nil
}
return 0, wrapErr(err, "exceeded retries")
return "", wrapErr(err, "exceeded retries")
}
func (h *EthLastValueCache) Stop() {
close(h.quit)
}
func (h *EthLastValueCache) Read() interface{} {
h.mutex.RLock()
defer h.mutex.RUnlock()
return h.value
func (h *EthLastValueCache) Read(ctx context.Context) (string, error) {
return h.cache.Get(ctx, h.key)
}
go/proxyd/methods.go
View file @ 2b34a1ee
......@@ -6,23 +6,17 @@ import (
"errors"
"fmt"
"sync"
"time"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/golang/snappy"
)
const numBlockConfirmations = 50
var (
cacheTTL = 5 * time.Second
errInvalidRPCParams = errors.New("invalid RPC params")
)
type RPCMethodHandler interface {
GetRPCMethod(context.Context, *RPCReq) (*RPCRes, error)
PutRPCMethod(context.Context, *RPCReq, *RPCRes, uint64) error
PutRPCMethod(context.Context, *RPCReq, *RPCRes) error
}
type StaticMethodHandler struct {
......@@ -42,7 +36,7 @@ func (e *StaticMethodHandler) GetRPCMethod(ctx context.Context, req *RPCReq) (*R
return cache, nil
}
func (e *StaticMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes, blockNumSync uint64) error {
func (e *StaticMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes) error {
e.m.Lock()
if e.cache == nil {
e.cache = copyRes(res)
......@@ -52,8 +46,9 @@ func (e *StaticMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res
}
type EthGetBlockByNumberMethodHandler struct {
cache Cache
getLatestBlockNumFn GetLatestBlockNumFn
cache Cache
getLatestBlockNumFn GetLatestBlockNumFn
numBlockConfirmations int
}
func (e *EthGetBlockByNumberMethodHandler) cacheKey(req *RPCReq) string {
......@@ -80,7 +75,7 @@ func (e *EthGetBlockByNumberMethodHandler) GetRPCMethod(ctx context.Context, req
return getImmutableRPCResponse(ctx, e.cache, key, req)
}
func (e *EthGetBlockByNumberMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes, blockNumberSync uint64) error {
func (e *EthGetBlockByNumberMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes) error {
if ok, err := e.cacheable(req); !ok || err != nil {
return err
}
......@@ -101,7 +96,7 @@ func (e *EthGetBlockByNumberMethodHandler) PutRPCMethod(ctx context.Context, req
if err != nil {
return err
}
if curBlock <= blockNum+numBlockConfirmations {
if curBlock <= blockNum+uint64(e.numBlockConfirmations) {
return nil
}
}
......@@ -111,8 +106,9 @@ func (e *EthGetBlockByNumberMethodHandler) PutRPCMethod(ctx context.Context, req
}
type EthGetBlockRangeMethodHandler struct {
cache Cache
getLatestBlockNumFn GetLatestBlockNumFn
cache Cache
getLatestBlockNumFn GetLatestBlockNumFn
numBlockConfirmations int
}
func (e *EthGetBlockRangeMethodHandler) cacheKey(req *RPCReq) string {
......@@ -140,7 +136,7 @@ func (e *EthGetBlockRangeMethodHandler) GetRPCMethod(ctx context.Context, req *R
return getImmutableRPCResponse(ctx, e.cache, key, req)
}
func (e *EthGetBlockRangeMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes, blockNumberSync uint64) error {
func (e *EthGetBlockRangeMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes) error {
if ok, err := e.cacheable(req); !ok || err != nil {
return err
}
......@@ -158,7 +154,7 @@ func (e *EthGetBlockRangeMethodHandler) PutRPCMethod(ctx context.Context, req *R
if err != nil {
return err
}
if curBlock <= startNum+numBlockConfirmations {
if curBlock <= startNum+uint64(e.numBlockConfirmations) {
return nil
}
}
......@@ -167,7 +163,7 @@ func (e *EthGetBlockRangeMethodHandler) PutRPCMethod(ctx context.Context, req *R
if err != nil {
return err
}
if curBlock <= endNum+numBlockConfirmations {
if curBlock <= endNum+uint64(e.numBlockConfirmations) {
return nil
}
}
......@@ -177,58 +173,66 @@ func (e *EthGetBlockRangeMethodHandler) PutRPCMethod(ctx context.Context, req *R
}
type EthCallMethodHandler struct {
cache Cache
getLatestBlockNumFn GetLatestBlockNumFn
cache Cache
getLatestBlockNumFn GetLatestBlockNumFn
numBlockConfirmations int
}
func (e *EthCallMethodHandler) cacheKey(req *RPCReq) string {
type ethCallParams struct {
From string `json:"from"`
To string `json:"to"`
Gas string `json:"gas"`
GasPrice string `json:"gasPrice"`
Value string `json:"value"`
Data string `json:"data"`
}
var input []json.RawMessage
if err := json.Unmarshal(req.Params, &input); err != nil {
return ""
}
if len(input) != 2 {
return ""
}
var blockTag string
if err := json.Unmarshal(input[1], &blockTag); err != nil {
return ""
}
// The eth_call cache is used as a LVC. Only the latest calls are cached as these are used the most
if blockTag != "latest" {
return ""
}
var params ethCallParams
if err := json.Unmarshal(input[0], &params); err != nil {
return ""
func (e *EthCallMethodHandler) cacheable(params *ethCallParams, blockTag string) bool {
if isBlockDependentParam(blockTag) {
return false
}
if params.From != "" || params.Gas != "" {
return ""
return false
}
if params.Value != "" && params.Value != "0x0" {
return ""
return false
}
// ensure the order is consistent
keyParams := fmt.Sprintf("%s:%s", params.To, params.Data)
return true
}
func (e *EthCallMethodHandler) cacheKey(params *ethCallParams, blockTag string) string {
keyParams := fmt.Sprintf("%s:%s:%s", params.To, params.Data, blockTag)
return fmt.Sprintf("method:eth_call:%s", keyParams)
}
func (e *EthCallMethodHandler) GetRPCMethod(ctx context.Context, req *RPCReq) (*RPCRes, error) {
key := e.cacheKey(req)
return getBlockDependentCachedRPCResponse(ctx, e.cache, e.getLatestBlockNumFn, key, req)
params, blockTag, err := decodeEthCallParams(req)
if err != nil {
return nil, err
}
if !e.cacheable(params, blockTag) {
return nil, nil
}
key := e.cacheKey(params, blockTag)
return getImmutableRPCResponse(ctx, e.cache, key, req)
}
func (e *EthCallMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes, blockNumberSync uint64) error {
key := e.cacheKey(req)
return putBlockDependentCachedRPCResponse(ctx, e.cache, key, res, blockNumberSync)
func (e *EthCallMethodHandler) PutRPCMethod(ctx context.Context, req *RPCReq, res *RPCRes) error {
params, blockTag, err := decodeEthCallParams(req)
if err != nil {
return err
}
if !e.cacheable(params, blockTag) {
return nil
}
if blockTag != "earliest" {
curBlock, err := e.getLatestBlockNumFn(ctx)
if err != nil {
return err
}
blockNum, err := decodeBlockInput(blockTag)
if err != nil {
return err
}
if curBlock <= blockNum+uint64(e.numBlockConfirmations) {
return nil
}
}
key := e.cacheKey(params, blockTag)
return putImmutableRPCResponse(ctx, e.cache, key, req, res)
}
type EthBlockNumberMethodHandler struct {
......@@ -243,7 +247,7 @@ func (e *EthBlockNumberMethodHandler) GetRPCMethod(ctx context.Context, req *RPC
return makeRPCRes(req, hexutil.EncodeUint64(blockNum)), nil
}
func (e *EthBlockNumberMethodHandler) PutRPCMethod(context.Context, *RPCReq, *RPCRes, uint64) error {
func (e *EthBlockNumberMethodHandler) PutRPCMethod(context.Context, *RPCReq, *RPCRes) error {
return nil
}
......@@ -259,7 +263,7 @@ func (e *EthGasPriceMethodHandler) GetRPCMethod(ctx context.Context, req *RPCReq
return makeRPCRes(req, hexutil.EncodeUint64(gasPrice)), nil
}
func (e *EthGasPriceMethodHandler) PutRPCMethod(context.Context, *RPCReq, *RPCRes, uint64) error {
func (e *EthGasPriceMethodHandler) PutRPCMethod(context.Context, *RPCReq, *RPCRes) error {
return nil
}
......@@ -319,6 +323,34 @@ func decodeBlockInput(input string) (uint64, error) {
return hexutil.DecodeUint64(input)
}
type ethCallParams struct {
From string `json:"from"`
To string `json:"to"`
Gas string `json:"gas"`
GasPrice string `json:"gasPrice"`
Value string `json:"value"`
Data string `json:"data"`
}
func decodeEthCallParams(req *RPCReq) (*ethCallParams, string, error) {
var input []json.RawMessage
if err := json.Unmarshal(req.Params, &input); err != nil {
return nil, "", err
}
if len(input) != 2 {
return nil, "", fmt.Errorf("invalid eth_call parameters")
}
params := new(ethCallParams)
if err := json.Unmarshal(input[0], params); err != nil {
return nil, "", err
}
var blockTag string
if err := json.Unmarshal(input[1], &blockTag); err != nil {
return nil, "", err
}
return params, blockTag, nil
}
func validBlockInput(input string) bool {
if input == "earliest" || input == "pending" || input == "latest" {
return true
......@@ -355,100 +387,30 @@ func copyRes(res *RPCRes) *RPCRes {
}
}
type CachedRPC struct {
BlockNum uint64 `json:"blockNum"`
Res *RPCRes `json:"res"`
Expiration int64 `json:"expiration"` // in millis since epoch
}
func (c *CachedRPC) Encode() []byte {
return mustMarshalJSON(c)
}
func (c *CachedRPC) Decode(b []byte) error {
return json.Unmarshal(b, c)
}
func (c *CachedRPC) ExpirationTime() time.Time {
return time.Unix(0, c.Expiration*int64(time.Millisecond))
}
func getImmutableRPCResponse(ctx context.Context, cache Cache, key string, req *RPCReq) (*RPCRes, error) {
encodedVal, err := cache.Get(ctx, key)
val, err := cache.Get(ctx, key)
if err != nil {
return nil, err
}
if encodedVal == "" {
if val == "" {
return nil, nil
}
val, err := snappy.Decode(nil, []byte(encodedVal))
if err != nil {
return nil, err
}
res := new(RPCRes)
if err := json.Unmarshal(val, res); err != nil {
var result interface{}
if err := json.Unmarshal([]byte(val), &result); err != nil {
return nil, err
}
res.ID = req.ID
return res, nil
return &RPCRes{
JSONRPC: req.JSONRPC,
Result: result,
ID: req.ID,
}, nil
}
func putImmutableRPCResponse(ctx context.Context, cache Cache, key string, req *RPCReq, res *RPCRes) error {
if key == "" {
return nil
}
val := mustMarshalJSON(res)
encodedVal := snappy.Encode(nil, val)
return cache.Put(ctx, key, string(encodedVal))
}
func getBlockDependentCachedRPCResponse(ctx context.Context, cache Cache, getLatestBlockNumFn GetLatestBlockNumFn, key string, req *RPCReq) (*RPCRes, error) {
encodedVal, err := cache.Get(ctx, key)
if err != nil {
return nil, err
}
if encodedVal == "" {
return nil, nil
}
val, err := snappy.Decode(nil, []byte(encodedVal))
if err != nil {
return nil, err
}
item := new(CachedRPC)
if err := json.Unmarshal(val, item); err != nil {
return nil, err
}
curBlockNum, err := getLatestBlockNumFn(ctx)
if err != nil {
return nil, err
}
expired := time.Now().After(item.ExpirationTime())
if curBlockNum > item.BlockNum && expired {
// Remove the key now to avoid stale entries from biasing the LRU list
// TODO: be careful removing keys once there are multiple proxyd instances
return nil, cache.Remove(ctx, key)
} else if curBlockNum < item.BlockNum { /* desync: reorgs, backend failover, slow sequencer I/O, etc */
return nil, nil
}
res := item.Res
res.ID = req.ID
return res, nil
}
func putBlockDependentCachedRPCResponse(ctx context.Context, cache Cache, key string, res *RPCRes, blockNumberSync uint64) error {
if key == "" {
return nil
}
item := CachedRPC{
BlockNum: blockNumberSync,
Res: res,
Expiration: time.Now().Add(cacheTTL).UnixNano() / int64(time.Millisecond),
}
val := item.Encode()
encodedVal := snappy.Encode(nil, val)
return cache.Put(ctx, key, string(encodedVal))
val := mustMarshalJSON(res.Result)
return cache.Put(ctx, key, string(val))
}
go/proxyd/metrics.go
View file @ 2b34a1ee
......@@ -145,7 +145,7 @@ var (
requestPayloadSizesGauge = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Name: "request_payload_sizes",
Help: "Gauge of client request payload sizes.",
Help: "Histogram of client request payload sizes.",
Buckets: PayloadSizeBuckets,
}, []string{
"auth",
......@@ -154,7 +154,7 @@ var (
responsePayloadSizesGauge = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: MetricsNamespace,
Name: "response_payload_sizes",
Help: "Gauge of client response payload sizes.",
Help: "Histogram of client response payload sizes.",
Buckets: PayloadSizeBuckets,
}, []string{
"auth",
......@@ -176,6 +176,22 @@ var (
"method",
})
lvcErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: MetricsNamespace,
Name: "lvc_errors_total",
Help: "Count of lvc errors.",
}, []string{
"key",
})
lvcPollTimeGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: MetricsNamespace,
Name: "lvc_poll_time_gauge",
Help: "Gauge of lvc poll time.",
}, []string{
"key",
})
rpcSpecialErrors = []string{
"nonce too low",
"gas price too high",
......
go/proxyd/proxyd.go
View file @ 2b34a1ee
......@@ -5,9 +5,9 @@ import (
"crypto/tls"
"errors"
"fmt"
"math/big"
"net/http"
"os"
"strconv"
"time"
"github.com/ethereum/go-ethereum/ethclient"
......@@ -111,12 +111,7 @@ func Start(config *Config) (func(), error) {
opts = append(opts, WithStrippedTrailingXFF())
}
opts = append(opts, WithProxydIP(os.Getenv("PROXYD_IP")))
back, err := NewBackend(name, rpcURL, wsURL, lim, opts...)
if err != nil {
return nil, err
}
back.Start()
defer back.Stop()
back := NewBackend(name, rpcURL, wsURL, lim, opts...)
backendNames = append(backendNames, name)
backendsByName[name] = back
log.Info("configured backend", "name", name, "rpc_url", rpcURL, "ws_url", wsURL)
......@@ -169,9 +164,18 @@ func Start(config *Config) (func(), error) {
}
}
var rpcCache RPCCache
stopLVCs := make(chan struct{})
var (
rpcCache RPCCache
blockNumLVC *EthLastValueCache
gasPriceLVC *EthLastValueCache
)
if config.Cache.Enabled {
var (
cache Cache
blockNumFn GetLatestBlockNumFn
gasPriceFn GetLatestGasPriceFn
)
if config.Cache.BlockSyncRPCURL == "" {
return nil, fmt.Errorf("block sync node required for caching")
}
......@@ -180,7 +184,6 @@ func Start(config *Config) (func(), error) {
return nil, err
}
var cache Cache
if redisURL != "" {
if cache, err = newRedisCache(redisURL); err != nil {
return nil, err
......@@ -195,9 +198,10 @@ func Start(config *Config) (func(), error) {
return nil, err
}
defer ethClient.Close()
blockNumFn := makeGetLatestBlockNumFn(ethClient, stopLVCs)
gasPriceFn := makeGetLatestGasPriceFn(ethClient, stopLVCs)
rpcCache = newRPCCache(cache, blockNumFn, gasPriceFn)
blockNumLVC, blockNumFn = makeGetLatestBlockNumFn(ethClient, cache)
gasPriceLVC, gasPriceFn = makeGetLatestGasPriceFn(ethClient, cache)
rpcCache = newRPCCache(newCacheWithCompression(cache), blockNumFn, gasPriceFn, config.Cache.NumBlockConfirmations)
}
srv := NewServer(
......@@ -250,8 +254,12 @@ func Start(config *Config) (func(), error) {
return func() {
log.Info("shutting down proxyd")
// TODO(inphi): Stop LVCs here
close(stopLVCs)
if blockNumLVC != nil {
blockNumLVC.Stop()
}
if gasPriceLVC != nil {
gasPriceLVC.Stop()
}
srv.Shutdown()
if err := lim.FlushBackendWSConns(backendNames); err != nil {
log.Error("error flushing backend ws conns", "err", err)
......@@ -285,38 +293,38 @@ func configureBackendTLS(cfg *BackendConfig) (*tls.Config, error) {
return tlsConfig, nil
}
func makeGetLatestBlockNumFn(client *ethclient.Client, quit <-chan struct{}) GetLatestBlockNumFn {
lvc := newLVC(client, func(ctx context.Context, c *ethclient.Client) (interface{}, error) {
return c.BlockNumber(ctx)
})
func makeUint64LastValueFn(client *ethclient.Client, cache Cache, key string, updater lvcUpdateFn) (*EthLastValueCache, func(context.Context) (uint64, error)) {
lvc := newLVC(client, cache, key, updater)
lvc.Start()
go func() {
<-quit
lvc.Stop()
}()
return func(ctx context.Context) (uint64, error) {
value := lvc.Read()
if value == nil {
return 0, fmt.Errorf("block number is unavailable")
}
return value.(uint64), nil
return lvc, func(ctx context.Context) (uint64, error) {
value, err := lvc.Read(ctx)
if err != nil {
return 0, err
}
if value == "" {
return 0, fmt.Errorf("%s is unavailable", key)
}
valueUint, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return 0, err
}
return valueUint, nil
}
}
func makeGetLatestGasPriceFn(client *ethclient.Client, quit <-chan struct{}) GetLatestGasPriceFn {
lvc := newLVC(client, func(ctx context.Context, c *ethclient.Client) (interface{}, error) {
return c.SuggestGasPrice(ctx)
func makeGetLatestBlockNumFn(client *ethclient.Client, cache Cache) (*EthLastValueCache, GetLatestBlockNumFn) {
return makeUint64LastValueFn(client, cache, "lvc:block_number", func(ctx context.Context, c *ethclient.Client) (string, error) {
blockNum, err := c.BlockNumber(ctx)
return strconv.FormatUint(blockNum, 10), err
})
}
func makeGetLatestGasPriceFn(client *ethclient.Client, cache Cache) (*EthLastValueCache, GetLatestGasPriceFn) {
return makeUint64LastValueFn(client, cache, "lvc:gas_price", func(ctx context.Context, c *ethclient.Client) (string, error) {
gasPrice, err := c.SuggestGasPrice(ctx)
if err != nil {
return "", err
}
return gasPrice.String(), nil
})
lvc.Start()
go func() {
<-lvc.quit
lvc.Stop()
}()
return func(ctx context.Context) (uint64, error) {
value := lvc.Read()
if value == nil {
return 0, fmt.Errorf("gas price is unavailable")
}
return value.(*big.Int).Uint64(), nil
}
}
go/proxyd/server.go
View file @ 2b34a1ee
......@@ -218,9 +218,7 @@ func (s *Server) handleSingleRPC(ctx context.Context, req *RPCReq) *RPCRes {
return backendRes
}
// NOTE: We call into the specific backend here to ensure that the RPCRes is synchronized with the blockNum.
var blockNum uint64
backendRes, blockNum, err = s.backendGroups[group].Forward(ctx, req)
backendRes, err = s.backendGroups[group].Forward(ctx, req)
if err != nil {
log.Error(
"error forwarding RPC request",
......@@ -232,7 +230,7 @@ func (s *Server) handleSingleRPC(ctx context.Context, req *RPCReq) *RPCRes {
}
if backendRes.Error == nil {
if err = s.cache.PutRPC(ctx, req, backendRes, blockNum); err != nil {
if err = s.cache.PutRPC(ctx, req, backendRes); err != nil {
log.Warn(
"cache put error",
"req_id", GetReqID(ctx),
......@@ -427,6 +425,6 @@ func (n *NoopRPCCache) GetRPC(context.Context, *RPCReq) (*RPCRes, error) {
return nil, nil
}
func (n *NoopRPCCache) PutRPC(context.Context, *RPCReq, *RPCRes, uint64) error {
func (n *NoopRPCCache) PutRPC(context.Context, *RPCReq, *RPCRes) error {
return nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment