563 lines
17 KiB
Go
563 lines
17 KiB
Go
package health
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ledgerwatch/erigon/common/hexutil"
|
|
"github.com/ledgerwatch/erigon/rpc"
|
|
)
|
|
|
|
type netApiStub struct {
|
|
response hexutil.Uint
|
|
error error
|
|
}
|
|
|
|
func (n *netApiStub) PeerCount(_ context.Context) (hexutil.Uint, error) {
|
|
return n.response, n.error
|
|
}
|
|
|
|
type ethApiStub struct {
|
|
blockResult map[string]interface{}
|
|
blockError error
|
|
syncingResult interface{}
|
|
syncingError error
|
|
}
|
|
|
|
func (e *ethApiStub) GetBlockByNumber(_ context.Context, _ rpc.BlockNumber, _ bool) (map[string]interface{}, error) {
|
|
return e.blockResult, e.blockError
|
|
}
|
|
|
|
func (e *ethApiStub) Syncing(_ context.Context) (interface{}, error) {
|
|
return e.syncingResult, e.syncingError
|
|
}
|
|
|
|
func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
|
|
cases := []struct {
|
|
headers []string
|
|
netApiResponse hexutil.Uint
|
|
netApiError error
|
|
ethApiBlockResult map[string]interface{}
|
|
ethApiBlockError error
|
|
ethApiSyncingResult interface{}
|
|
ethApiSyncingError error
|
|
expectedStatusCode int
|
|
expectedBody map[string]string
|
|
}{
|
|
// 0 - sync check enabled - syncing
|
|
{
|
|
headers: []string{"synced"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusOK,
|
|
expectedBody: map[string]string{
|
|
synced: "HEALTHY",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 1 - sync check enabled - not syncing
|
|
{
|
|
headers: []string{"synced"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: struct{}{},
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "ERROR: not synced",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 2 - sync check enabled - error checking sync
|
|
{
|
|
headers: []string{"synced"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: struct{}{},
|
|
ethApiSyncingError: errors.New("problem checking sync"),
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "ERROR: problem checking sync",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 3 - peer count enabled - good request
|
|
{
|
|
headers: []string{"min_peer_count1"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusOK,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "HEALTHY",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 4 - peer count enabled - not enough peers
|
|
{
|
|
headers: []string{"min_peer_count10"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "ERROR: not enough peers: 1 (minimum 10)",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 5 - peer count enabled - error checking peers
|
|
{
|
|
headers: []string{"min_peer_count10"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: errors.New("problem checking peers"),
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "ERROR: problem checking peers",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 6 - peer count enabled - badly formed request
|
|
{
|
|
headers: []string{"min_peer_countABC"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: make(map[string]interface{}),
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "ERROR: strconv.Atoi: parsing \"abc\": invalid syntax",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 7 - block check - all ok
|
|
{
|
|
headers: []string{"check_block10"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{"test": struct{}{}},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusOK,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "HEALTHY",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 8 - block check - no block found
|
|
{
|
|
headers: []string{"check_block10"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "ERROR: no known block with number 10 (a hex)",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 9 - block check - error checking block
|
|
{
|
|
headers: []string{"check_block10"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{},
|
|
ethApiBlockError: errors.New("problem checking block"),
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "ERROR: problem checking block",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 10 - block check - badly formed request
|
|
{
|
|
headers: []string{"check_blockABC"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "ERROR: strconv.Atoi: parsing \"abc\": invalid syntax",
|
|
maxSecondsBehind: "DISABLED",
|
|
},
|
|
},
|
|
// 11 - seconds check - all ok
|
|
{
|
|
headers: []string{"max_seconds_behind60"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{
|
|
"timestamp": time.Now().Add(1 * time.Second).Unix(),
|
|
},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusOK,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "HEALTHY",
|
|
},
|
|
},
|
|
// 12 - seconds check - too old
|
|
{
|
|
headers: []string{"max_seconds_behind60"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{
|
|
"timestamp": uint64(time.Now().Add(1 * time.Hour).Unix()),
|
|
},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "ERROR: timestamp too old: got ts:",
|
|
},
|
|
},
|
|
// 13 - seconds check - less than 0 seconds
|
|
{
|
|
headers: []string{"max_seconds_behind-1"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{
|
|
"timestamp": uint64(time.Now().Add(1 * time.Hour).Unix()),
|
|
},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "ERROR: bad header value",
|
|
},
|
|
},
|
|
// 14 - seconds check - badly formed request
|
|
{
|
|
headers: []string{"max_seconds_behindABC"},
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
synced: "DISABLED",
|
|
minPeerCount: "DISABLED",
|
|
checkBlock: "DISABLED",
|
|
maxSecondsBehind: "ERROR: strconv.Atoi: parsing \"abc\": invalid syntax",
|
|
},
|
|
},
|
|
// 15 - all checks - report ok
|
|
{
|
|
headers: []string{"synced", "check_block10", "min_peer_count1", "max_seconds_behind60"},
|
|
netApiResponse: hexutil.Uint(10),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{
|
|
"timestamp": time.Now().Add(1 * time.Second).Unix(),
|
|
},
|
|
ethApiBlockError: nil,
|
|
ethApiSyncingResult: false,
|
|
ethApiSyncingError: nil,
|
|
expectedStatusCode: http.StatusOK,
|
|
expectedBody: map[string]string{
|
|
synced: "HEALTHY",
|
|
minPeerCount: "HEALTHY",
|
|
checkBlock: "HEALTHY",
|
|
maxSecondsBehind: "HEALTHY",
|
|
},
|
|
},
|
|
}
|
|
|
|
for idx, c := range cases {
|
|
w := httptest.NewRecorder()
|
|
r, err := http.NewRequest(http.MethodGet, "http://localhost:9090/health", nil)
|
|
if err != nil {
|
|
t.Errorf("%v: creating request: %v", idx, err)
|
|
}
|
|
|
|
for _, header := range c.headers {
|
|
r.Header.Add("X-ERIGON-HEALTHCHECK", header)
|
|
}
|
|
|
|
netAPI := rpc.API{
|
|
Namespace: "",
|
|
Version: "",
|
|
Service: &netApiStub{
|
|
response: c.netApiResponse,
|
|
error: c.netApiError,
|
|
},
|
|
Public: false,
|
|
}
|
|
|
|
ethAPI := rpc.API{
|
|
Namespace: "",
|
|
Version: "",
|
|
Service: ðApiStub{
|
|
blockResult: c.ethApiBlockResult,
|
|
blockError: c.ethApiBlockError,
|
|
syncingResult: c.ethApiSyncingResult,
|
|
syncingError: c.ethApiSyncingError,
|
|
},
|
|
Public: false,
|
|
}
|
|
|
|
apis := make([]rpc.API, 2)
|
|
apis[0] = netAPI
|
|
apis[1] = ethAPI
|
|
|
|
ProcessHealthcheckIfNeeded(w, r, apis)
|
|
|
|
result := w.Result()
|
|
if result.StatusCode != c.expectedStatusCode {
|
|
t.Errorf("%v: expected status code: %v, but got: %v", idx, c.expectedStatusCode, result.StatusCode)
|
|
}
|
|
|
|
bodyBytes, err := io.ReadAll(result.Body)
|
|
if err != nil {
|
|
t.Errorf("%v: reading response body: %s", idx, err)
|
|
}
|
|
|
|
var body map[string]string
|
|
err = json.Unmarshal(bodyBytes, &body)
|
|
if err != nil {
|
|
t.Errorf("%v: unmarshalling the response body: %s", idx, err)
|
|
}
|
|
result.Body.Close()
|
|
|
|
for k, v := range c.expectedBody {
|
|
val, found := body[k]
|
|
if !found {
|
|
t.Errorf("%v: expected the key: %s to be in the response body but it wasn't there", idx, k)
|
|
}
|
|
if !strings.Contains(val, v) {
|
|
t.Errorf("%v: expected the response body key: %s to contain: %s, but it contained: %s", idx, k, v, val)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessHealthcheckIfNeeded_RequestBody(t *testing.T) {
|
|
cases := []struct {
|
|
body string
|
|
netApiResponse hexutil.Uint
|
|
netApiError error
|
|
ethApiBlockResult map[string]interface{}
|
|
ethApiBlockError error
|
|
expectedStatusCode int
|
|
expectedBody map[string]string
|
|
}{
|
|
// 0 - happy path
|
|
{
|
|
body: "{\"min_peer_count\": 1, \"known_block\": 123}",
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{"test": struct{}{}},
|
|
ethApiBlockError: nil,
|
|
expectedStatusCode: http.StatusOK,
|
|
expectedBody: map[string]string{
|
|
"healthcheck_query": "HEALTHY",
|
|
"min_peer_count": "HEALTHY",
|
|
"check_block": "HEALTHY",
|
|
},
|
|
},
|
|
// 1 - bad request body
|
|
{
|
|
body: "{\"min_peer_count\" 1, \"known_block\": 123}",
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{"test": struct{}{}},
|
|
ethApiBlockError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
"healthcheck_query": "ERROR:",
|
|
"min_peer_count": "DISABLED",
|
|
"check_block": "DISABLED",
|
|
},
|
|
},
|
|
// 2 - min peers - error from api
|
|
{
|
|
body: "{\"min_peer_count\": 1, \"known_block\": 123}",
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: errors.New("problem getting peers"),
|
|
ethApiBlockResult: map[string]interface{}{"test": struct{}{}},
|
|
ethApiBlockError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
"healthcheck_query": "HEALTHY",
|
|
"min_peer_count": "ERROR: problem getting peers",
|
|
"check_block": "HEALTHY",
|
|
},
|
|
},
|
|
// 3 - min peers - not enough peers
|
|
{
|
|
body: "{\"min_peer_count\": 10, \"known_block\": 123}",
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{"test": struct{}{}},
|
|
ethApiBlockError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
"healthcheck_query": "HEALTHY",
|
|
"min_peer_count": "ERROR: not enough peers",
|
|
"check_block": "HEALTHY",
|
|
},
|
|
},
|
|
// 4 - check block - no block
|
|
{
|
|
body: "{\"min_peer_count\": 1, \"known_block\": 123}",
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{},
|
|
ethApiBlockError: nil,
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
"healthcheck_query": "HEALTHY",
|
|
"min_peer_count": "HEALTHY",
|
|
"check_block": "ERROR: no known block with number ",
|
|
},
|
|
},
|
|
// 5 - check block - error getting block info
|
|
{
|
|
body: "{\"min_peer_count\": 1, \"known_block\": 123}",
|
|
netApiResponse: hexutil.Uint(1),
|
|
netApiError: nil,
|
|
ethApiBlockResult: map[string]interface{}{},
|
|
ethApiBlockError: errors.New("problem getting block"),
|
|
expectedStatusCode: http.StatusInternalServerError,
|
|
expectedBody: map[string]string{
|
|
"healthcheck_query": "HEALTHY",
|
|
"min_peer_count": "HEALTHY",
|
|
"check_block": "ERROR: problem getting block",
|
|
},
|
|
},
|
|
}
|
|
|
|
for idx, c := range cases {
|
|
w := httptest.NewRecorder()
|
|
r, err := http.NewRequest(http.MethodGet, "http://localhost:9090/health", nil)
|
|
if err != nil {
|
|
t.Errorf("%v: creating request: %v", idx, err)
|
|
}
|
|
|
|
r.Body = io.NopCloser(strings.NewReader(c.body))
|
|
|
|
netAPI := rpc.API{
|
|
Namespace: "",
|
|
Version: "",
|
|
Service: &netApiStub{
|
|
response: c.netApiResponse,
|
|
error: c.netApiError,
|
|
},
|
|
Public: false,
|
|
}
|
|
|
|
ethAPI := rpc.API{
|
|
Namespace: "",
|
|
Version: "",
|
|
Service: ðApiStub{
|
|
blockResult: c.ethApiBlockResult,
|
|
blockError: c.ethApiBlockError,
|
|
},
|
|
Public: false,
|
|
}
|
|
|
|
apis := make([]rpc.API, 2)
|
|
apis[0] = netAPI
|
|
apis[1] = ethAPI
|
|
|
|
ProcessHealthcheckIfNeeded(w, r, apis)
|
|
|
|
result := w.Result()
|
|
if result.StatusCode != c.expectedStatusCode {
|
|
t.Errorf("%v: expected status code: %v, but got: %v", idx, c.expectedStatusCode, result.StatusCode)
|
|
}
|
|
|
|
bodyBytes, err := io.ReadAll(result.Body)
|
|
if err != nil {
|
|
t.Errorf("%v: reading response body: %s", idx, err)
|
|
}
|
|
|
|
var body map[string]string
|
|
err = json.Unmarshal(bodyBytes, &body)
|
|
if err != nil {
|
|
t.Errorf("%v: unmarshalling the response body: %s", idx, err)
|
|
}
|
|
result.Body.Close()
|
|
|
|
for k, v := range c.expectedBody {
|
|
val, found := body[k]
|
|
if !found {
|
|
t.Errorf("%v: expected the key: %s to be in the response body but it wasn't there", idx, k)
|
|
}
|
|
if !strings.Contains(val, v) {
|
|
t.Errorf("%v: expected the response body key: %s to contain: %s, but it contained: %s", idx, k, v, val)
|
|
}
|
|
}
|
|
}
|
|
}
|