adds certman package with k8s volume reload fixes

op-service/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/ethereum-optimism/optimism/op-node v0.10.12
 	github.com/ethereum-optimism/optimism/op-signer v0.1.0
 	github.com/ethereum/go-ethereum v1.10.26
+	github.com/fsnotify/fsnotify v1.6.0
 	github.com/prometheus/client_golang v1.13.0
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli v1.22.9
@@ -27,7 +28,6 @@ require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/dyson/certman v0.3.0 // indirect
 	github.com/fjl/memsize v0.0.1 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
 	github.com/go-kit/kit v0.10.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect

op-service/rpc/server.go

@@ -183,7 +183,7 @@ func (b *Server) Start() error {
 	errCh := make(chan error, 1)
 	go func() {
 		if b.tls != nil {
-			if err := b.httpServer.ListenAndServeTLS(b.tls.CLIConfig.TLSCert, b.tls.CLIConfig.TLSKey); err != nil {
+			if err := b.httpServer.ListenAndServeTLS("", ""); err != nil {
 				errCh <- err
 			}
 		} else {

op-service/tls/certman/certman.go

+// Package certman provides live reloading of the certificate and key
+// files used by the standard library http.Server. It defines a type,
+// certMan, with methods watching and getting the files.
+// Only valid certificate and key pairs are loaded and an optional
+// logger can be passed to certman for logging providing it implements
+// the logger interface.
+package certman
+
+import (
+	"crypto/tls"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/fsnotify/fsnotify"
+)
+
+// A CertMan represents a certificate manager able to watch certificate
+// and key pairs for changes.
+type CertMan struct {
+	mu       sync.RWMutex
+	certFile string
+	keyFile  string
+	keyPair  *tls.Certificate
+	watcher  *fsnotify.Watcher
+	watching chan bool
+	log      log.Logger
+}
+
+// New creates a new certMan. The certFile and the keyFile
+// are both paths to the location of the files. Relative and
+// absolute paths are accepted.
+func New(logger log.Logger, certFile, keyFile string) (*CertMan, error) {
+	var err error
+	certFile, err = filepath.Abs(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyFile, err = filepath.Abs(keyFile)
+	if err != nil {
+		return nil, err
+	}
+	cm := &CertMan{
+		mu:       sync.RWMutex{},
+		certFile: certFile,
+		keyFile:  keyFile,
+		log:      logger,
+	}
+	return cm, nil
+}
+
+// Watch starts watching for changes to the certificate
+// and key files. On any change the certificate and key
+// are reloaded. If there is an issue the load will fail
+// and the old (if any) certificates and keys will continue
+// to be used.
+func (cm *CertMan) Watch() error {
+	var err error
+	if cm.watcher, err = fsnotify.NewWatcher(); err != nil {
+		return fmt.Errorf("certman: can't create watcher: %w", err)
+	}
+
+	certPath := path.Dir(cm.certFile)
+	keyPath := path.Dir(cm.keyFile)
+
+	if err = cm.watcher.Add(certPath); err != nil {
+		return fmt.Errorf("certman: can't watch %s: %w", certPath, err)
+	}
+	if keyPath != certPath {
+		if err = cm.watcher.Add(keyPath); err != nil {
+			return fmt.Errorf("certman: can't watch %s: %w", certPath, err)
+		}
+	}
+	if err := cm.load(); err != nil {
+		cm.log.Error("certman: can't load cert or key file", "err", err)
+	}
+	cm.log.Info("certman: watching for cert and key change")
+	cm.watching = make(chan bool)
+	go cm.run()
+	return nil
+}
+
+func (cm *CertMan) load() error {
+	keyPair, err := tls.LoadX509KeyPair(cm.certFile, cm.keyFile)
+	if err == nil {
+		cm.mu.Lock()
+		cm.keyPair = &keyPair
+		cm.mu.Unlock()
+		cm.log.Info("certman: certificate and key loaded")
+	}
+	return err
+}
+
+func (cm *CertMan) run() {
+	cm.log.Info("certman: running")
+
+	ticker := time.NewTicker(2 * time.Second)
+	files := []string{cm.certFile, cm.keyFile}
+	reload := time.Time{}
+
+loop:
+	for {
+		select {
+		case <-cm.watching:
+			cm.log.Info("watching triggered; break loop")
+			break loop
+		case <-ticker.C:
+			if !reload.IsZero() && time.Now().After(reload) {
+				reload = time.Time{}
+				cm.log.Info("certman: reloading")
+				if err := cm.load(); err != nil {
+					cm.log.Error("certman: can't load cert or key file", "err", err)
+				}
+			}
+		case event := <-cm.watcher.Events:
+			for _, f := range files {
+				if event.Name == f ||
+					strings.HasSuffix(event.Name, "/..data") { // kubernetes secrets mount
+					// we wait a couple seconds in case the cert and key don't update atomically
+					cm.log.Info(fmt.Sprintf("%s was modified, queue reload", f))
+					reload = time.Now().Add(2 * time.Second)
+				}
+			}
+		case err := <-cm.watcher.Errors:
+			cm.log.Error("certman: error watching files", "err", err)
+		}
+	}
+	cm.log.Info("certman: stopped watching")
+	cm.watcher.Close()
+	ticker.Stop()
+}
+
+// GetCertificate returns the loaded certificate for use by
+// the GetCertificate field in tls.Config.
+func (cm *CertMan) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+	return cm.keyPair, nil
+}
+
+// GetClientCertificate returns the loaded certificate for use by
+// the GetClientCertificate field in tls.Config.
+func (cm *CertMan) GetClientCertificate(hello *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+	return cm.keyPair, nil
+}
+
+// Stop tells certMan to stop watching for changes to the
+// certificate and key files.
+func (cm *CertMan) Stop() {
+	cm.watching <- false
+}

op-service/tls/certman/certman_test.go

+// Copyright 2017 Dyson Simmons. All rights reserved.
+// Use of this source code is governed by a MIT
+// license that can be found in the LICENSE file.
+
+package certman_test
+
+import (
+	"crypto/tls"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-service/tls/certman"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func TestValidPair(t *testing.T) {
+	cm, err := certman.New(log.Root(), "./testdata/server1.crt", "./testdata/server1.key")
+	if err != nil {
+		t.Errorf("could not create certman: %v", err)
+	}
+	if err := cm.Watch(); err != nil {
+		t.Errorf("could not watch files: %v", err)
+	}
+}
+
+func TestInvalidPair(t *testing.T) {
+	cm, err := certman.New(log.Root(), "./testdata/server1.crt", "./testdata/server2.key")
+	if err != nil {
+		t.Errorf("could not create certman: %v", err)
+	}
+	if err := cm.Watch(); err != nil {
+		t.Errorf("could not watch files: %v", err)
+	}
+}
+
+func TestCertificateNotFound(t *testing.T) {
+	cm, err := certman.New(log.Root(), "./testdata/nothere.crt", "./testdata/server2.key")
+	if err != nil {
+		t.Errorf("could not create certman: %v", err)
+	}
+	if err := cm.Watch(); err != nil {
+		if !strings.HasPrefix(err.Error(), "certman: can't watch cert file: ") {
+			t.Errorf("unexpected watch error: %v", err)
+		}
+	}
+}
+
+func TestKeyNotFound(t *testing.T) {
+	cm, err := certman.New(log.Root(), "./testdata/server1.crt", "./testdata/nothere.key")
+	if err != nil {
+		t.Errorf("could not create certman: %v", err)
+	}
+	if err := cm.Watch(); err != nil {
+		if !strings.HasPrefix(err.Error(), "certman: can't watch key file: ") {
+			t.Errorf("unexpected watch error: %v", err)
+		}
+	}
+}
+
+func TestGetCertificate(t *testing.T) {
+	cm, err := certman.New(log.Root(), "./testdata/server1.crt", "./testdata/server1.key")
+	if err != nil {
+		t.Errorf("could not create certman: %v", err)
+	}
+	if err := cm.Watch(); err != nil {
+		t.Errorf("could not watch files: %v", err)
+	}
+	hello := &tls.ClientHelloInfo{}
+	cmCert, err := cm.GetCertificate(hello)
+	if err != nil {
+		t.Error("could not get certman certificate")
+	}
+	expectedCert, _ := tls.LoadX509KeyPair("./testdata/server1.crt", "./testdata/server1.key")
+	if err != nil {
+		t.Errorf("could not load certificate and key files to test: %v", err)
+	}
+	if !reflect.DeepEqual(cmCert.Certificate, expectedCert.Certificate) {
+		t.Errorf("certman certificate doesn't match expected certificate")
+	}
+}

op-service/tls/certman/testdata/README

+Test certificates generated with the following command:
+
+openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 -sha256 -keyout server.key -out server.crt
\ No newline at end of file

op-service/tls/certman/testdata/server.crt

+-----BEGIN CERTIFICATE-----
+MIIDWjCCAkKgAwIBAgIJAJiO53P0WQTzMA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV
+BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg
+Q29tcGFueSBMdGQwHhcNMTcwODA0MTAzMzIyWhcNMTgwODA0MTAzMzIyWjBCMQsw
+CQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZh
+dWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+sYk+rsOElxPB5gPH8Vg/RFzdBzDJesI4VCXYcSCl0Ek/lO4AAKjgRssjmZDWWXcZ
+3/Z1UlO4cRZy6zaSllFM07WXX3SQ54iBy3lS6NgBcHiWVfKrtXz/dQrswSPBnGEH
+PVBmd+xHcEv4wqYnYYVtJuIcGY1P/i14h1ogpEW22cnCVYJwqmjxrD4UQ8vSeTf9
+YwVCbKXZ5T+eiNhfQOkCG0rClqvfvZiMgBBlLYvJrc9bELsqIbNGVD4CrkYjZGEc
+ToMrb/wZhqLxlcs/iXKI0/579+9vvc44/pi2MFU0nvhoSnEgopKKPy83SwBogpiH
++mDcVgMiEwbXHUyNAFbz/wIDAQABo1MwUTAdBgNVHQ4EFgQULtTUM8bHdg4jhRT3
+9piSLGyTSsUwHwYDVR0jBBgwFoAULtTUM8bHdg4jhRT39piSLGyTSsUwDwYDVR0T
+AQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAchsCFqaRslfgR0r0CoOJsX/H
+gEfqYHaL8/yJtUUqCRgyN5VtP1Cxzet8GVsAJaIKimeUCXshhaq94JQRZqxMFxJD
+sD9nfWVByorkSO9tmq+1FCWRzfau1AFsJxR6J1hpIbRfcfoi9HG4QnoJwULK/2yh
+DEXwnmgsCHTcNnj2U9F2vMLydHEKtmtMNe/S0Z7fHw1qlmqXgXmN5a/KTEPVBQm8
+eM6AhNlzrMqUPc8IiiZ32eAdgR2eAuhLnTCdnHwnHafpep72hSjwoUXsDjWNA/xW
+/lHx4jVL8muLP5G9YHZWzJwMijLqIlnryyyi+IfKAn4ANW/k44Lxm5hmRXh4UA==
+-----END CERTIFICATE-----

op-service/tls/certman/testdata/server.key

+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCxiT6uw4SXE8Hm
+A8fxWD9EXN0HMMl6wjhUJdhxIKXQST+U7gAAqOBGyyOZkNZZdxnf9nVSU7hxFnLr
+NpKWUUzTtZdfdJDniIHLeVLo2AFweJZV8qu1fP91CuzBI8GcYQc9UGZ37EdwS/jC
+pidhhW0m4hwZjU/+LXiHWiCkRbbZycJVgnCqaPGsPhRDy9J5N/1jBUJspdnlP56I
+2F9A6QIbSsKWq9+9mIyAEGUti8mtz1sQuyohs0ZUPgKuRiNkYRxOgytv/BmGovGV
+yz+JcojT/nv372+9zjj+mLYwVTSe+GhKcSCikoo/LzdLAGiCmIf6YNxWAyITBtcd
+TI0AVvP/AgMBAAECggEATZpnau8N+xfozsliUa24YgKRnv4FZAKXqrisRq71q/kI
+sOnj2GX5Oxi6o/q6p3q3Nb2+hNERs2UTsJs3MjuxcG1VEKWcXYi+65lJ03vwDSC4
+3jLoObm81IWE/dvKWrfS+Us2rz757y1WPIdyeV9gWfnGPKkXiUyI/ek4kXXjuoiL
+WqYeWlhe2mGVxQs5DfxBigvN0th2yfmeQJTLXpoJkCDmSTS3vnvKJfiH/mflLm/B
+51lgKEziqvu53JmgYmhswsgRnLtyZ4yjksMroJYPKCTA3RbDud+J6SO01ok+hiNy
+QcO/j+1zlFQlWqAE4BSB1IXbW/6ZQAdszPpX9l318QKBgQDaW+nq0FH7djne5838
+sqBO7pHGde2j6To8PuGIoskH86p8X4dgutNiingGnYoqK2qTKVTudkdu2ahwGI3P
+S3dflwcyJwAvTbjc3ATSoez+P0oUC62DQciMfqaCR+WYAoAR8cdHtSoMyMvLVhSs
+1h05AoalTybuT+spscV8hFBdlwKBgQDQI9VxKPt9VwhKf0W6K2IKbNxvHcu0UFvZ
+pCZKdxQLr7tP3+oo9fWn6R0sSX/aV+km9YQsn6keX0e4ayJHNfU6UwzNSMKWQZvs
+JF7OBuZnpZYX/1lSIUDOw8I29kcVtqCHjpgozbFhJSQzJDDcPvRFmkPBI74sgA1z
+xuH7xcc52QKBgDPX7snZfB2ADG1oC/gbUQRskB/Wj/2CuljjdRjDzYcdyzSMWdAV
+i2qyBZ1MeilY9YzLG2cingMrmlpC+ihleooviX3W1Kxmf6Wwd1SrLWGQFT59J00q
+qTryNwZnm5NjxJR+GxpjYQB4DCrS3UXL8FRAzUcia9PZFbRoiMLvh0UxAoGBAIjZ
+prL6cTBeEvN4bw4TDCkynlTo0FDELUASL6LyXFm6t3uzC7DW1ygJm8bMpKWY+5FE
+CB2W9IkluHBG8IjFr3Ejvd0To+1LQgundjYcT02CkAdDOyVG++d2yrF8iAx8wVuf
+o+fgJmprEzwU5ZNKSS2iWj4ZFCcKIs4my9rQlUcxAoGBAMiqpes+Ka3ofb8sxpzj
+xAB2oOub0X93a3xQ4xv5VLDysA7+rv1CDwuYKlOKTjGFwg3CwvbnE2EKkKI9BNgh
+UHAdpVXWOck5ubD5EB6vp7xWvzu6AA4KyhyRJaRL1GdSLFl9JDg5YvHskVxumt+L
+/GgTbnT/P3+JOhCdc0H0XhGq
+-----END PRIVATE KEY-----

op-service/tls/certman/testdata/server1.crt

+-----BEGIN CERTIFICATE-----
+MIIDWjCCAkKgAwIBAgIJALkZ2SqlmTT6MA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV
+BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg
+Q29tcGFueSBMdGQwHhcNMTcwODA1MTU1MjIyWhcNMTgwODA1MTU1MjIyWjBCMQsw
+CQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZh
+dWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+p3vZAGYjBC4kPf6jqh6NYCWMVe8Db5IOHYECtGxBVOhS5DnuXPm0LUT9UCoUfPeB
+mU/1LZk41J70tpwl2IHhSIFqbG+fhNlRlUAA9CIHeRa2RVkFxH7I7xt2FhwU2WsX
+vi+GEiOKoZ2mC2cuZ5t8zWKinW+Hd8pL270OboQ980gThfF2ugzoXlyYVa+MrE5z
+wjLa3lQoKkqzo54/gKwK+KycXYoljmf2Q0++sSsDAqZlFIYknfct4+ST4TvGRqSw
+nPxQcJ33MXDpbnGcTBPXqWxXhvocd+QrOF4Rn7fVRLWZqYoFchSnt4R3qsoPuB8q
+WdZn+McT6qqgLUUtRy72PQIDAQABo1MwUTAdBgNVHQ4EFgQU83D1oNnMvjHU5q3E
+KIJUtrHMh94wHwYDVR0jBBgwFoAU83D1oNnMvjHU5q3EKIJUtrHMh94wDwYDVR0T
+AQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEADByFIBVqK7b2ZJVqWzQlY7pC
+DbR4bh+z8hwP/VU/JjlR0IdFj1tWLLPq5hmCWScntsmI8AvThCTbZgwogptvbeuA
+n2QRuSb3LMZBtq22XdDTPCdM4CQblinbXR3ePmz2ZpF7xxQMz8/IafeoZaaF2w4l
+PfHrf4ID89dMOq13MAhiSFmKeyElx2iGGnhsGQzhcoTbhDCW/HK3Zr/BN+uFyBgr
+PcV0H+VLOS7/XVnz5wbIiqTfnx0NhzQzw91zY2dXVOvNadD6QpNN5Abwxo9x16TP
+SfqKsjPvxk06wvchNAkopBLsqjLMovgKYMMbolVmFeeZVvNEnY19RoBuneJncA==
+-----END CERTIFICATE-----

op-service/tls/certman/testdata/server1.key

+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCne9kAZiMELiQ9
+/qOqHo1gJYxV7wNvkg4dgQK0bEFU6FLkOe5c+bQtRP1QKhR894GZT/UtmTjUnvS2
+nCXYgeFIgWpsb5+E2VGVQAD0Igd5FrZFWQXEfsjvG3YWHBTZaxe+L4YSI4qhnaYL
+Zy5nm3zNYqKdb4d3ykvbvQ5uhD3zSBOF8Xa6DOheXJhVr4ysTnPCMtreVCgqSrOj
+nj+ArAr4rJxdiiWOZ/ZDT76xKwMCpmUUhiSd9y3j5JPhO8ZGpLCc/FBwnfcxcOlu
+cZxME9epbFeG+hx35Cs4XhGft9VEtZmpigVyFKe3hHeqyg+4HypZ1mf4xxPqqqAt
+RS1HLvY9AgMBAAECggEAGE5R7Mvl0wp7OgAFcn/ilox8dFAumHeC0udRJCv9wzvA
+I90Aab/XVSaI+KRSutwUk9JVy5tL8xdqfkHlACnBLwuRDVGZvebn/xf9y3BQ01Ln
+euLzglPAB2td1NGYeQEgvfoZo/JCgTfmzAraYjDfiNMCtIRmDY1vOuGSAZnxf6e/
+C7LB7XdKdGijUgjvHOv82DUbdt5RD/RL9chvk+i9esdjJIxS3YaSOHclV7FSjRqe
+f5e9MrPKuSdijYHqHOcxX3hztOrLjL7J4gVPRvjGaWbRmdU79m2oFlwfGlLTU+w6
+zlzIu5FqKD375rchkOKw3W8uSyPCRsdnmZxT+CZKIQKBgQDPHiXmGWlTBdcBcgaK
+4PIZltAZbA+5MdDDJPDZp9b5gS8/0mXoqOvvC2PeOkrv96l3nrVoTbfOxxutftjO
+Xh2OfplpNkxABR8WqY7mKaHR4aphlq1QtxcCcOlmAbmo9Eo03Vhir7q913upgEnY
+jzEoX9b41JEJBTR6Mfbfx3b6+QKBgQDPAw2ZuBqiyPmozCDqKnP2yiOJz6wkunqI
+rUkZLizN+U9kByiq/NzoDGo6BNchs2XpVhoIt3ixC/xzzhDUkn7V7ZCn8BkEdj/T
+jgZyupMJ1wNQK9Du3GaYPRnQ1I3OrfA0oOx1ZVoOOKOEDAEAfhzSW8GWicO3305o
+6/4Tq4gCZQKBgQC18vEuS+KX+chgz6/pryVfz3ou6xyA/786v6gKPYUAGTnN4mJ+
+Wm8xx5rLLgCJANPSbw1EfQndUFMDPizuVgW3GYZhxD6F+znNadVMYwRyYcGRC5Jk
+FwPStCiF4TwdrcXG3TB5OZFelv9e74FwCpMPueobHHnxJ65rLpuHCS5/2QKBgBDF
+AYwLUvUO7NKUvrHZgI1kcJ6QWTSceqKpzvsgN3b0FE9ZGR1I4KhXoR9UFw1e2Amf
+9PnxyvAktW24KrrdpzKzTP2dwJkQ7zi3D6SpopGwfk83TXScHB+HC5lULqyogIXy
+51TXQgVW50AiLM6aaMFNt4/3VwiFKXfsbievxJPVAoGADlJV011pkxlF2wZxpvDT
+RGIWCtT3G3GksNPxTkLDsuBnJ38YE0p95MivBP3Q7kdVhfgOzG1AnWMeOUzcdzNM
+hXKW9EcyovPYrHpAcdwMXOj+WuU74l2tHLPJQ5+4zTKAc/4ZJ7TKMjZ21qRjqGHQ
+dxddYOr2kCOIFNreAaKD9Po=
+-----END PRIVATE KEY-----

op-service/tls/certman/testdata/server2.crt

+-----BEGIN CERTIFICATE-----
+MIIDWjCCAkKgAwIBAgIJAJiO53P0WQTzMA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV
+BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg
+Q29tcGFueSBMdGQwHhcNMTcwODA0MTAzMzIyWhcNMTgwODA0MTAzMzIyWjBCMQsw
+CQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZh
+dWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+sYk+rsOElxPB5gPH8Vg/RFzdBzDJesI4VCXYcSCl0Ek/lO4AAKjgRssjmZDWWXcZ
+3/Z1UlO4cRZy6zaSllFM07WXX3SQ54iBy3lS6NgBcHiWVfKrtXz/dQrswSPBnGEH
+PVBmd+xHcEv4wqYnYYVtJuIcGY1P/i14h1ogpEW22cnCVYJwqmjxrD4UQ8vSeTf9
+YwVCbKXZ5T+eiNhfQOkCG0rClqvfvZiMgBBlLYvJrc9bELsqIbNGVD4CrkYjZGEc
+ToMrb/wZhqLxlcs/iXKI0/579+9vvc44/pi2MFU0nvhoSnEgopKKPy83SwBogpiH
++mDcVgMiEwbXHUyNAFbz/wIDAQABo1MwUTAdBgNVHQ4EFgQULtTUM8bHdg4jhRT3
+9piSLGyTSsUwHwYDVR0jBBgwFoAULtTUM8bHdg4jhRT39piSLGyTSsUwDwYDVR0T
+AQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAchsCFqaRslfgR0r0CoOJsX/H
+gEfqYHaL8/yJtUUqCRgyN5VtP1Cxzet8GVsAJaIKimeUCXshhaq94JQRZqxMFxJD
+sD9nfWVByorkSO9tmq+1FCWRzfau1AFsJxR6J1hpIbRfcfoi9HG4QnoJwULK/2yh
+DEXwnmgsCHTcNnj2U9F2vMLydHEKtmtMNe/S0Z7fHw1qlmqXgXmN5a/KTEPVBQm8
+eM6AhNlzrMqUPc8IiiZ32eAdgR2eAuhLnTCdnHwnHafpep72hSjwoUXsDjWNA/xW
+/lHx4jVL8muLP5G9YHZWzJwMijLqIlnryyyi+IfKAn4ANW/k44Lxm5hmRXh4UA==
+-----END CERTIFICATE-----

op-service/tls/certman/testdata/server2.key

+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCxiT6uw4SXE8Hm
+A8fxWD9EXN0HMMl6wjhUJdhxIKXQST+U7gAAqOBGyyOZkNZZdxnf9nVSU7hxFnLr
+NpKWUUzTtZdfdJDniIHLeVLo2AFweJZV8qu1fP91CuzBI8GcYQc9UGZ37EdwS/jC
+pidhhW0m4hwZjU/+LXiHWiCkRbbZycJVgnCqaPGsPhRDy9J5N/1jBUJspdnlP56I
+2F9A6QIbSsKWq9+9mIyAEGUti8mtz1sQuyohs0ZUPgKuRiNkYRxOgytv/BmGovGV
+yz+JcojT/nv372+9zjj+mLYwVTSe+GhKcSCikoo/LzdLAGiCmIf6YNxWAyITBtcd
+TI0AVvP/AgMBAAECggEATZpnau8N+xfozsliUa24YgKRnv4FZAKXqrisRq71q/kI
+sOnj2GX5Oxi6o/q6p3q3Nb2+hNERs2UTsJs3MjuxcG1VEKWcXYi+65lJ03vwDSC4
+3jLoObm81IWE/dvKWrfS+Us2rz757y1WPIdyeV9gWfnGPKkXiUyI/ek4kXXjuoiL
+WqYeWlhe2mGVxQs5DfxBigvN0th2yfmeQJTLXpoJkCDmSTS3vnvKJfiH/mflLm/B
+51lgKEziqvu53JmgYmhswsgRnLtyZ4yjksMroJYPKCTA3RbDud+J6SO01ok+hiNy
+QcO/j+1zlFQlWqAE4BSB1IXbW/6ZQAdszPpX9l318QKBgQDaW+nq0FH7djne5838
+sqBO7pHGde2j6To8PuGIoskH86p8X4dgutNiingGnYoqK2qTKVTudkdu2ahwGI3P
+S3dflwcyJwAvTbjc3ATSoez+P0oUC62DQciMfqaCR+WYAoAR8cdHtSoMyMvLVhSs
+1h05AoalTybuT+spscV8hFBdlwKBgQDQI9VxKPt9VwhKf0W6K2IKbNxvHcu0UFvZ
+pCZKdxQLr7tP3+oo9fWn6R0sSX/aV+km9YQsn6keX0e4ayJHNfU6UwzNSMKWQZvs
+JF7OBuZnpZYX/1lSIUDOw8I29kcVtqCHjpgozbFhJSQzJDDcPvRFmkPBI74sgA1z
+xuH7xcc52QKBgDPX7snZfB2ADG1oC/gbUQRskB/Wj/2CuljjdRjDzYcdyzSMWdAV
+i2qyBZ1MeilY9YzLG2cingMrmlpC+ihleooviX3W1Kxmf6Wwd1SrLWGQFT59J00q
+qTryNwZnm5NjxJR+GxpjYQB4DCrS3UXL8FRAzUcia9PZFbRoiMLvh0UxAoGBAIjZ
+prL6cTBeEvN4bw4TDCkynlTo0FDELUASL6LyXFm6t3uzC7DW1ygJm8bMpKWY+5FE
+CB2W9IkluHBG8IjFr3Ejvd0To+1LQgundjYcT02CkAdDOyVG++d2yrF8iAx8wVuf
+o+fgJmprEzwU5ZNKSS2iWj4ZFCcKIs4my9rQlUcxAoGBAMiqpes+Ka3ofb8sxpzj
+xAB2oOub0X93a3xQ4xv5VLDysA7+rv1CDwuYKlOKTjGFwg3CwvbnE2EKkKI9BNgh
+UHAdpVXWOck5ubD5EB6vp7xWvzu6AA4KyhyRJaRL1GdSLFl9JDg5YvHskVxumt+L
+/GgTbnT/P3+JOhCdc0H0XhGq
+-----END PRIVATE KEY-----