Skip to content

Commit 4c9d908

Browse files
siyuanfoundationsiyuanfoundation
committed
add MemberDowngradeUpgrade failpoint
Signed-off-by: Siyuan Zhang <[email protected]>
1 parent ebb2b06 commit 4c9d908

File tree

5 files changed

+198
-98
lines changed

5 files changed

+198
-98
lines changed

‎tests/e2e/cluster_downgrade_test.go‎

Lines changed: 4 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,13 @@ package e2e
1616

1717
import (
1818
"context"
19-
"encoding/json"
2019
"fmt"
21-
"strings"
2220
"testing"
2321
"time"
2422

2523
"github.com/coreos/go-semver/semver"
2624
"github.com/stretchr/testify/assert"
2725
"github.com/stretchr/testify/require"
28-
"go.uber.org/zap"
2926

3027
"go.etcd.io/etcd/api/v3/version"
3128
"go.etcd.io/etcd/client/pkg/v3/fileutil"
@@ -86,7 +83,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
8683
var snapshotCount uint64 = 10
8784
epc := newCluster(t, clusterSize, snapshotCount)
8885
for i := 0; i < len(epc.Procs); i++ {
89-
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
86+
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
9087
Cluster: currentVersionStr,
9188
Server: version.Version,
9289
Storage: currentVersionStr,
@@ -117,7 +114,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
117114

118115
t.Log("Downgrade enabled, validating if cluster is ready for downgrade")
119116
for i := 0; i < len(epc.Procs); i++ {
120-
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
117+
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
121118
Cluster: lastClusterVersionStr,
122119
Server: version.Version,
123120
Storage: lastClusterVersionStr,
@@ -136,7 +133,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
136133
t.Log("All members downgraded, validating downgrade")
137134
e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")
138135
for i := 0; i < len(epc.Procs); i++ {
139-
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
136+
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
140137
Cluster: lastClusterVersionStr,
141138
Server: lastVersionStr,
142139
})
@@ -176,7 +173,7 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) {
176173

177174
t.Log("All members upgraded, validating upgrade")
178175
for i := 0; i < len(epc.Procs); i++ {
179-
validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
176+
e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
180177
Cluster: currentVersionStr,
181178
Server: version.Version,
182179
Storage: currentVersionStr,
@@ -227,27 +224,6 @@ func stopEtcd(t *testing.T, ep e2e.EtcdProcess) {
227224
require.NoError(t, err)
228225
}
229226

230-
func validateVersion(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, expect version.Versions) {
231-
testutils.ExecuteWithTimeout(t, 30*time.Second, func() {
232-
for {
233-
result, err := getMemberVersionByCurl(cfg, member)
234-
if err != nil {
235-
cfg.Logger.Warn("failed to get member version and retrying", zap.Error(err), zap.String("member", member.Config().Name))
236-
time.Sleep(time.Second)
237-
continue
238-
}
239-
cfg.Logger.Info("Comparing versions", zap.String("member", member.Config().Name), zap.Any("got", result), zap.Any("want", expect))
240-
if err := compareMemberVersion(expect, result); err != nil {
241-
cfg.Logger.Warn("Versions didn't match retrying", zap.Error(err), zap.String("member", member.Config().Name))
242-
time.Sleep(time.Second)
243-
continue
244-
}
245-
cfg.Logger.Info("Versions match", zap.String("member", member.Config().Name))
246-
break
247-
}
248-
})
249-
}
250-
251227
func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess {
252228
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
253229
defer cancel()
@@ -269,36 +245,6 @@ func leader(t *testing.T, epc *e2e.EtcdProcessCluster) e2e.EtcdProcess {
269245
return nil
270246
}
271247

272-
func compareMemberVersion(expect version.Versions, target version.Versions) error {
273-
if expect.Server != "" && expect.Server != target.Server {
274-
return fmt.Errorf("expect etcdserver version %v, but got %v", expect.Server, target.Server)
275-
}
276-
277-
if expect.Cluster != "" && expect.Cluster != target.Cluster {
278-
return fmt.Errorf("expect etcdcluster version %v, but got %v", expect.Cluster, target.Cluster)
279-
}
280-
281-
if expect.Storage != "" && expect.Storage != target.Storage {
282-
return fmt.Errorf("expect storage version %v, but got %v", expect.Storage, target.Storage)
283-
}
284-
return nil
285-
}
286-
287-
func getMemberVersionByCurl(cfg *e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess) (version.Versions, error) {
288-
args := e2e.CURLPrefixArgsCluster(cfg, member, "GET", e2e.CURLReq{Endpoint: "/version"})
289-
lines, err := e2e.RunUtilCompletion(args, nil)
290-
if err != nil {
291-
return version.Versions{}, err
292-
}
293-
294-
data := strings.Join(lines, "\n")
295-
result := version.Versions{}
296-
if err := json.Unmarshal([]byte(data), &result); err != nil {
297-
return version.Versions{}, fmt.Errorf("failed to unmarshal (%v): %w", data, err)
298-
}
299-
return result, nil
300-
}
301-
302248
func generateSnapshot(t *testing.T, snapshotCount uint64, cc *e2e.EtcdctlV3) {
303249
ctx, cancel := context.WithCancel(context.Background())
304250
defer cancel()

‎tests/framework/e2e/curl.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,8 @@ func CURLPut(clus *EtcdProcessCluster, req CURLReq) error {
123123
}
124124

125125
func CURLGet(clus *EtcdProcessCluster, req CURLReq) error {
126-
ctx, cancel := context.WithTimeout(context.Background(), req.timeoutDuration())
127-
defer cancel()
128-
129-
return SpawnWithExpectsContext(ctx, CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "GET", req), nil, req.Expected)
126+
member := clus.Procs[rand.Intn(clus.Cfg.ClusterSize)]
127+
return CURLGetFromMember(clus, member, req)
130128
}
131129

132130
func CURLGetFromMember(clus *EtcdProcessCluster, member EtcdProcess, req CURLReq) error {

‎tests/framework/e2e/util.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,13 @@ import (
2424
"testing"
2525
"time"
2626

27+
"github.com/coreos/go-semver/semver"
28+
"go.uber.org/zap"
29+
30+
"go.etcd.io/etcd/api/v3/version"
2731
"go.etcd.io/etcd/client/pkg/v3/testutil"
2832
"go.etcd.io/etcd/pkg/v3/expect"
33+
"go.etcd.io/etcd/tests/v3/framework/testutils"
2934
)
3035

3136
func WaitReadyExpectProc(ctx context.Context, exproc *expect.ExpectProcess, readyStrs []string) error {
@@ -177,3 +182,60 @@ func mergeEnvVariables(envVars map[string]string) []string {
177182

178183
return env
179184
}
185+
186+
func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdProcess, expect version.Versions) {
187+
testutils.ExecuteWithTimeout(t, 30*time.Second, func() {
188+
for {
189+
result, err := getMemberVersionByCurl(cfg, member)
190+
if err != nil {
191+
cfg.Logger.Warn("failed to get member version and retrying", zap.Error(err), zap.String("member", member.Config().Name))
192+
time.Sleep(time.Second)
193+
continue
194+
}
195+
cfg.Logger.Info("Comparing versions", zap.String("member", member.Config().Name), zap.Any("got", result), zap.Any("want", expect))
196+
if err := compareMemberVersion(expect, result); err != nil {
197+
cfg.Logger.Warn("Versions didn't match retrying", zap.Error(err), zap.String("member", member.Config().Name))
198+
time.Sleep(time.Second)
199+
continue
200+
}
201+
cfg.Logger.Info("Versions match", zap.String("member", member.Config().Name))
202+
break
203+
}
204+
})
205+
}
206+
207+
func majorMinorVersionsEqual(v1, v2 string) bool {
208+
ver1 := semver.New(v1)
209+
ver2 := semver.New(v2)
210+
return ver1.Major == ver2.Major && ver1.Minor == ver2.Minor
211+
}
212+
213+
func compareMemberVersion(expect version.Versions, target version.Versions) error {
214+
if expect.Server != "" && !majorMinorVersionsEqual(expect.Server, target.Server) {
215+
return fmt.Errorf("expect etcdserver version %v, but got %v", expect.Server, target.Server)
216+
}
217+
218+
if expect.Cluster != "" && !majorMinorVersionsEqual(expect.Cluster, target.Cluster) {
219+
return fmt.Errorf("expect etcdcluster version %v, but got %v", expect.Cluster, target.Cluster)
220+
}
221+
222+
if expect.Storage != "" && !majorMinorVersionsEqual(expect.Storage, target.Storage) {
223+
return fmt.Errorf("expect storage version %v, but got %v", expect.Storage, target.Storage)
224+
}
225+
return nil
226+
}
227+
228+
func getMemberVersionByCurl(cfg *EtcdProcessClusterConfig, member EtcdProcess) (version.Versions, error) {
229+
args := CURLPrefixArgsCluster(cfg, member, "GET", CURLReq{Endpoint: "/version"})
230+
lines, err := RunUtilCompletion(args, nil)
231+
if err != nil {
232+
return version.Versions{}, err
233+
}
234+
235+
data := strings.Join(lines, "\n")
236+
result := version.Versions{}
237+
if err := json.Unmarshal([]byte(data), &result); err != nil {
238+
return version.Versions{}, fmt.Errorf("failed to unmarshal (%v): %w", data, err)
239+
}
240+
return result, nil
241+
}

0 commit comments

Comments
 (0)