Unverified Commit 64476b01 authored by Lemon-miaow's avatar Lemon-miaow
Browse files

fix(store): 新 pod 首次连库在网络策略放行前重试

parent 7b91f7c6

cmd/felis/api.go

+1 −1
Changes for cmd/felis/api.go: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -100,7 +100,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { 


	// Before anything serves: an api on a schema it was not built for answers with

	// errors, or writes rows the other version cannot read.

	drv, err := openStore(ctx, cfg.Database.URL, false)

	drv, err := openPodStore(ctx, cfg.Database.URL, "api", stderr)

	if err != nil {

		fmt.Fprintf(stderr, "felis api: open database: %v\n", err)

		return 1

cmd/felis/backup.go

+1 −2
Changes for cmd/felis/backup.go: 1 added line, 2 removed lines.
Original line number Diff line number Diff line
@@ -15,7 +15,6 @@ import ( 
	"felis.lolicon.best/internal/config"

	"felis.lolicon.best/internal/naming"

	"felis.lolicon.best/internal/reaper"

	"felis.lolicon.best/internal/store"

	ctrl "sigs.k8s.io/controller-runtime"

)
@@ -100,7 +99,7 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int { 
			len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))

	}



	drv, err := store.Open(ctx, cfg.Database.URL)

	drv, err := openPodStore(ctx, cfg.Database.URL, "backup", stderr)

	if err != nil {

		fmt.Fprintf(stderr, "felis backup: open database: %v\n", err)

		return 1

cmd/felis/migrate.go

+28 −0
Changes for cmd/felis/migrate.go: 28 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ import ( 
	"flag"

	"fmt"

	"io"

	"time"



	"felis.lolicon.best/internal/config"

	"felis.lolicon.best/internal/dbbackup"
@@ -138,6 +139,33 @@ func openStore(ctx context.Context, url string, allowPending bool) (*store.Postg 
	if err != nil {

		return nil, err

	}

	return checkSchema(ctx, drv, allowPending)

}



// podDBWindow and podDBInterval bound how long a pod that has just started retries its

// first database dial while the network policy has yet to admit it

// (store.OpenRetrying). A minute is far past the sync lag and far inside every Job's

// deadline. Vars so a test can shrink them.

var (

	podDBWindow   = time.Minute

	podDBInterval = time.Second

)



// openPodStore is openStore for felis-api and the reaper and backup Jobs, whose first

// dial comes milliseconds after their pod starts.

func openPodStore(ctx context.Context, url, prog string, stderr io.Writer) (*store.PostgresDriver, error) {

	drv, err := store.OpenRetrying(ctx, url, podDBWindow, podDBInterval, func(err error) {

		fmt.Fprintf(stderr, "felis %s: %v; retrying (a pod that has just started waits for the network policy to admit it)\n", prog, err)

	})

	if err != nil {

		return nil, err

	}

	return checkSchema(ctx, drv, false)

}



// checkSchema closes drv and fails when its schema is not the one this build was

// written against (see openStore).

func checkSchema(ctx context.Context, drv *store.PostgresDriver, allowPending bool) (*store.PostgresDriver, error) {

	s, err := store.ReadSchema(ctx, drv)

	if err == nil {

		err = s.Err()

cmd/felis/migrate_test.go

0 → 100644
+48 −0
Changes for cmd/felis/migrate_test.go: 48 added lines, 0 removed lines.
Original line number Diff line number Diff line 
package main



import (

	"bytes"

	"context"

	"errors"

	"fmt"

	"net"

	"strings"

	"syscall"

	"testing"

	"time"

)



// openPodStore retries a refused first dial for podDBWindow, saying so on stderr

// under the calling command's name each time.

func TestOpenPodStoreRetriesARefusedDial(t *testing.T) {

	ln, err := net.Listen("tcp", "127.0.0.1:0")

	if err != nil {

		t.Fatal(err)

	}

	addr := ln.Addr().String()

	ln.Close()



	window, interval := podDBWindow, podDBInterval

	podDBWindow, podDBInterval = 200*time.Millisecond, 20*time.Millisecond

	t.Cleanup(func() { podDBWindow, podDBInterval = window, interval })



	var stderr bytes.Buffer

	start := time.Now()

	_, err = openPodStore(context.Background(), fmt.Sprintf("postgres://felis@%s/felis?sslmode=disable", addr), "reaper", &stderr)

	if !errors.Is(err, syscall.ECONNREFUSED) {

		t.Fatalf("err = %v, want a refused dial", err)

	}

	if elapsed := time.Since(start); elapsed < podDBWindow {

		t.Fatalf("gave up after %s, inside the %s window", elapsed, podDBWindow)

	}

	lines := strings.Split(strings.TrimSuffix(stderr.String(), "\n"), "\n")

	if len(lines) < 2 || len(lines) > 11 {

		t.Fatalf("%d retry lines in a 200ms window at 20ms:\n%s", len(lines), stderr.String())

	}

	for _, l := range lines {

		if !strings.HasPrefix(l, "felis reaper: failed to connect to `user=felis database=felis`: ") ||

			!strings.HasSuffix(l, "; retrying (a pod that has just started waits for the network policy to admit it)") {

			t.Fatalf("retry line %q", l)

		}

	}

}

cmd/felis/reaper.go

+1 −1
Changes for cmd/felis/reaper.go: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -78,7 +78,7 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int { 
		return 1

	}



	drv, err := openStore(ctx, cfg.Database.URL, false)

	drv, err := openPodStore(ctx, cfg.Database.URL, "reaper", stderr)

	if err != nil {

		fmt.Fprintf(stderr, "felis reaper: open database: %v\n", err)

		return 1